opennlp-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jo...@apache.org
Subject svn commit: r1570210 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: chunker/ChunkSampleSequenceStream.java chunker/ChunkerME.java chunker/ChunkerModel.java ml/BeamSearch.java ml/model/SequenceClassificationModel.java
Date Thu, 20 Feb 2014 14:53:17 GMT
Author: joern
Date: Thu Feb 20 14:53:16 2014
New Revision: 1570210

URL: http://svn.apache.org/r1570210
Log:
OPENNLP-641 Added initial sequence classification support

Added:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java

Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java?rev=1570210&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
(added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
Thu Feb 20 14:53:16 2014
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.chunker;
+
+import java.io.IOException;
+
+import opennlp.tools.ml.model.AbstractModel;
+import opennlp.tools.ml.model.Event;
+import opennlp.tools.ml.model.Sequence;
+import opennlp.tools.ml.model.SequenceStream;
+import opennlp.tools.util.ObjectStream;
+
+public class ChunkSampleSequenceStream implements SequenceStream {
+
+  private final ObjectStream<ChunkSample> samples;
+  private final ChunkerContextGenerator contextGenerator;
+
+  public ChunkSampleSequenceStream(ObjectStream<ChunkSample> samples,
+      ChunkerContextGenerator contextGenerator) {
+    this.samples = samples;
+    this.contextGenerator = contextGenerator;
+  }
+
+  @Override
+  public Sequence read() throws IOException {
+    ChunkSample sample = samples.read();
+    
+    if (sample != null) {
+      String sentence[] = sample.getSentence();
+      String tags[] = sample.getTags();
+      Event[] events = new Event[sentence.length];
+      
+      for (int i=0; i < sentence.length; i++) {
+  
+        // it is safe to pass the tags as previous tags because
+        // the context generator does not look for non predicted tags
+        String[] context = contextGenerator.getContext(i, sentence, tags, null);
+  
+        events[i] = new Event(tags[i], context);
+      }
+      return new Sequence<ChunkSample>(events,sample);
+    }
+    
+    return null;  
+  }
+
+  @Override
+  public Event[] updateContext(Sequence sequence, AbstractModel model) {
+    // TODO: Should be implemented for Perceptron sequence learning ...
+    return null;
+  }
+  
+  @Override
+  public void reset() throws IOException, UnsupportedOperationException {
+    samples.reset();
+  }
+
+  @Override
+  public void close() throws IOException {
+    samples.close();
+  }
+
+}

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java Thu Feb
20 14:53:16 2014
@@ -22,10 +22,15 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import opennlp.tools.ml.EventTrainer;
+import opennlp.tools.ml.SequenceTrainer;
+import opennlp.tools.ml.TrainerFactory;
+import opennlp.tools.ml.TrainerFactory.TrainerType;
 import opennlp.tools.ml.model.Event;
 import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.ml.model.SequenceClassificationModel;
 import opennlp.tools.ml.model.TrainUtil;
-import opennlp.tools.util.BeamSearch;
+import opennlp.tools.postag.POSSampleSequenceStream;
 import opennlp.tools.util.ObjectStream;
 import opennlp.tools.util.Sequence;
 import opennlp.tools.util.SequenceValidator;
@@ -40,17 +45,15 @@ public class ChunkerME implements Chunke
 
   public static final int DEFAULT_BEAM_SIZE = 10;
 
-  /**
-   * The beam used to search for sequences of chunk tag assignments.
-   */
-  protected BeamSearch<String> beam;
-
   private Sequence bestSequence;
 
   /**
    * The model used to assign chunk tags to a sequence of tokens.
    */
-  protected MaxentModel model;
+  protected SequenceClassificationModel<String> model;
+
+  private ChunkerContextGenerator contextGenerator;
+  private SequenceValidator<String> sequenceValidator;
 
   /**
    * Initializes the current instance with the specified model and
@@ -64,10 +67,20 @@ public class ChunkerME implements Chunke
    * @deprecated Use {@link #ChunkerME(ChunkerModel, int)} instead 
    *    and use the {@link ChunkerFactory} to configure the {@link SequenceValidator} and
{@link ChunkerContextGenerator}.
    */
+  @Deprecated
   public ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<String> sequenceValidator,
       ChunkerContextGenerator contextGenerator) {
-    this.model = model.getChunkerModel();
-    beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator,
0);
+    
+    this.sequenceValidator = sequenceValidator;
+    this.contextGenerator = contextGenerator;
+    
+    if (model.getChunkerModel() != null) {
+      this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+          model.getChunkerModel(), 0);
+    }
+    else {
+      this.model = model.getChunkerSequenceModel();
+    }
   }
   
   /**
@@ -82,12 +95,13 @@ public class ChunkerME implements Chunke
    * @deprecated Use {@link #ChunkerME(ChunkerModel, int)} instead 
    *    and use the {@link ChunkerFactory} to configure the {@link SequenceValidator}.
    */
+  @Deprecated
   public ChunkerME(ChunkerModel model, int beamSize,
       SequenceValidator<String> sequenceValidator) {
     this(model, beamSize, sequenceValidator,
         new DefaultChunkerContextGenerator());
   }
-
+  
   /**
    * Initializes the current instance with the specified model and
    * the specified beam size.
@@ -96,10 +110,18 @@ public class ChunkerME implements Chunke
    * @param beamSize The size of the beam that should be used when decoding sequences.
    */
   public ChunkerME(ChunkerModel model, int beamSize) {
-    this.model = model.getChunkerModel();
-    ChunkerContextGenerator contextGenerator = model.getFactory().getContextGenerator();
-    SequenceValidator<String> sequenceValidator = model.getFactory().getSequenceValidator();
-    beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator,
0);
+    
+   contextGenerator = model.getFactory().getContextGenerator();
+   sequenceValidator = model.getFactory().getSequenceValidator();
+    // beam = new BeamSearch<String>(beamSize, contextGenerator, this.model, sequenceValidator,
0);
+    
+    if (model.getChunkerModel() != null) {
+      this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+          model.getChunkerModel(), 0);
+    }
+    else {
+      this.model = model.getChunkerSequenceModel();
+    }
   }
   
   /**
@@ -115,12 +137,13 @@ public class ChunkerME implements Chunke
   @Deprecated
   public List<String> chunk(List<String> toks, List<String> tags) {
     bestSequence =
-        beam.bestSequence(toks.toArray(new String[toks.size()]), new Object[] { tags.toArray(new
String[tags.size()]) });
+        model.bestSequence(toks.toArray(new String[toks.size()]), new Object[] { tags.toArray(new
String[tags.size()]) },
+            contextGenerator, sequenceValidator);
     return bestSequence.getOutcomes();
   }
 
   public String[] chunk(String[] toks, String[] tags) {
-    bestSequence = beam.bestSequence(toks, new Object[] {tags});
+    bestSequence = model.bestSequence(toks, new Object[] {tags}, contextGenerator, sequenceValidator);
     List<String> c = bestSequence.getOutcomes();
     return c.toArray(new String[c.size()]);
   }
@@ -137,12 +160,13 @@ public class ChunkerME implements Chunke
   }
   
   public Sequence[] topKSequences(String[] sentence, String[] tags) {
-    return beam.bestSequences(DEFAULT_BEAM_SIZE, sentence,
-        new Object[] { tags });
+    return model.bestSequences(DEFAULT_BEAM_SIZE, sentence,
+        new Object[] { tags }, contextGenerator, sequenceValidator);
   }
 
   public Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore)
{
-    return beam.bestSequences(DEFAULT_BEAM_SIZE, sentence, new Object[] { tags },minSequenceScore);
+    return model.bestSequences(DEFAULT_BEAM_SIZE, sentence, new Object[] { tags }, minSequenceScore,
+        contextGenerator, sequenceValidator);
   }
 
   /**
@@ -171,12 +195,37 @@ public class ChunkerME implements Chunke
 
     Map<String, String> manifestInfoEntries = new HashMap<String, String>();
 
-    ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator());
-
-    MaxentModel maxentModel = TrainUtil.train(es, mlParams.getSettings(),
-        manifestInfoEntries);
+    TrainerType trainerType = TrainerFactory.getTrainerType(mlParams.getSettings());
+    
 
-    return new ChunkerModel(lang, maxentModel, manifestInfoEntries, factory);
+    MaxentModel chunkerModel = null;
+    SequenceClassificationModel<String> seqChunkerModel = null;
+    
+    if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) {
+      ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator());
+      EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams.getSettings(),
+          manifestInfoEntries);
+      chunkerModel = trainer.train(es);
+    }
+    else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) {
+      SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer(
+          mlParams.getSettings(), manifestInfoEntries);
+      
+      // TODO: This will probably cause issue, since the feature generator uses the outcomes
array
+      
+      ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator());
+      seqChunkerModel = trainer.train(ss);
+    }
+    else {
+      throw new IllegalArgumentException("Trainer type is not supported: " + trainerType);
 
+    }
+    
+    if (chunkerModel != null) {
+      return new ChunkerModel(lang, chunkerModel, manifestInfoEntries, factory);
+    }
+    else {
+      return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory);
+    }
   }
 
   /**

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java Thu
Feb 20 14:53:16 2014
@@ -31,6 +31,7 @@ import opennlp.tools.ml.model.AbstractMo
 import opennlp.tools.ml.model.BinaryFileDataReader;
 import opennlp.tools.ml.model.GenericModelReader;
 import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.ml.model.SequenceClassificationModel;
 import opennlp.tools.util.BaseToolFactory;
 import opennlp.tools.util.InvalidFormatException;
 import opennlp.tools.util.model.BaseModel;
@@ -55,6 +56,14 @@ public class ChunkerModel extends BaseMo
     this(languageCode, chunkerModel, manifestInfoEntries, new ChunkerFactory());
   }
   
+  public ChunkerModel(String languageCode, SequenceClassificationModel<String> chunkerModel,
+      Map<String, String> manifestInfoEntries, ChunkerFactory factory) {
+    super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
+    artifactMap.put(CHUNKER_MODEL_ENTRY_NAME, chunkerModel);
+    checkArtifactMap();
+  }
+
+  
   public ChunkerModel(String languageCode, MaxentModel chunkerModel,
       Map<String, String> manifestInfoEntries, ChunkerFactory factory) {
     super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
@@ -97,7 +106,21 @@ public class ChunkerModel extends BaseMo
   }
 
   public MaxentModel getChunkerModel() {
-    return (MaxentModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+    if (artifactMap.get(CHUNKER_MODEL_ENTRY_NAME) instanceof MaxentModel) {
+      return (MaxentModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+    }
+    else {
+      return null;
+    }
+  }
+  
+  public SequenceClassificationModel<String> getChunkerSequenceModel() {
+    if (artifactMap.get(CHUNKER_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel)
{
+      return (SequenceClassificationModel) artifactMap.get(CHUNKER_MODEL_ENTRY_NAME);
+    }
+    else {
+      return null;
+    }
   }
   
   @Override

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java Thu Feb 20
14:53:16 2014
@@ -80,7 +80,7 @@ public class BeamSearch<T> implements Se
    * @return The top ranked sequence of outcomes or null if no sequence could be found
    */
   public Sequence[] bestSequences(int numSequences, T[] sequence,
-      Object[] additionalContext, BeamSearchContextGenerator<T> cg, SequenceValidator<T>
validator) {
+      Object[] additionalContext, double minSequenceScore, BeamSearchContextGenerator<T>
cg, SequenceValidator<T> validator) {
 
     Heap<Sequence> prev = new ListHeap<Sequence>(size);
     Heap<Sequence> next = new ListHeap<Sequence>(size);
@@ -126,9 +126,9 @@ public class BeamSearch<T> implements Se
           String out = model.getOutcome(p);
            if (validator.validSequence(i, sequence, outcomes, out)) {
             Sequence ns = new Sequence(top, out, scores[p]);
-            // if (ns.getScore() > minSequenceScore) {
+            if (ns.getScore() > minSequenceScore) {
               next.add(ns);
-            // }
+            }
            }
         }
 
@@ -137,9 +137,9 @@ public class BeamSearch<T> implements Se
             String out = model.getOutcome(p);
             if (validator.validSequence(i, sequence, outcomes, out)) {
               Sequence ns = new Sequence(top, out, scores[p]);
-              // if (ns.getScore() > minSequenceScore) {
+              if (ns.getScore() > minSequenceScore) {
                 next.add(ns);
-              //}
+              }
             }
           }
         }
@@ -162,6 +162,11 @@ public class BeamSearch<T> implements Se
     return topSequences;
   }
   
+  public Sequence[] bestSequences(int numSequences, T[] sequence,
+      Object[] additionalContext, BeamSearchContextGenerator<T> cg, SequenceValidator<T>
validator) {
+    return bestSequences(numSequences, sequence, additionalContext, zeroLog, cg, validator);
+  }
+  
   public Sequence bestSequence(T[] sequence, Object[] additionalContext,
       BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator) {
     Sequence sequences[] =  bestSequences(1, sequence, additionalContext, cg, validator);

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java?rev=1570210&r1=1570209&r2=1570210&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java
(original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceClassificationModel.java
Thu Feb 20 14:53:16 2014
@@ -42,6 +42,19 @@ public interface SequenceClassificationM
    */
   Sequence bestSequence(T[] sequence, Object[] additionalContext,
       BeamSearchContextGenerator<T> cg, SequenceValidator<T> validator);
+
+  /**
+   * Finds the n most probable sequences.
+   * 
+   * @param sequence
+   * @param additionalContext
+   * @param cg
+   * @param validator
+   * 
+   * @return
+   */
+  Sequence[] bestSequences(int numSequences, T[] sequence,
+      Object[] additionalContext, double minSequenceScore, BeamSearchContextGenerator<T>
cg, SequenceValidator<T> validator);
   
   /**
    * Finds the n most probable sequences.



Mime
View raw message