opennlp-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lina abu jaradeh <linabu...@hotmail.com>
Subject Problem with my project
Date Sun, 18 May 2014 19:30:44 GMT
I am trying to train en-ner-location.bin file using opennlp in java The thing is i got the
training text file in the following format <START:location> Fontana <END> .<START:location>
Palo Verde <END> .
<START:location> Picacho <END> .
I stored it in a text file called citytrain.txt the file contains 120770 line of city namesthen
i used the following code to train the fileimport java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Collections;

import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.NameSampleDataStream;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;

public class TrainNames {	 
		@SuppressWarnings("deprecation")
		public void TrainNames() throws IOException{
			//File baseDir = new File("src/test/resources");
		    //File destDir = new File("target");
		    //<start id="ne-train"/>
		    File inFile = new File("citytrain.txt");
		    NameSampleDataStream nss = new NameSampleDataStream( //<co id="co.opennlp.name.initnamestream"/>
		      new PlainTextByLineStream(
		        new java.io.FileReader(inFile)));

		    int iterations = 100;
		    int cutoff = 5;
		    TokenNameFinderModel model = NameFinderME.train( //<co id="co.opennlp.name.train"/>
		        "en", // language
		        "location", // type
		        nss, 
		        (AdaptiveFeatureGenerator) null,
		        Collections.<String,Object>emptyMap(),
		        iterations,
		        cutoff);
		    
		    File outFile = new File("en-ner-locationNews.bin");
		    FileOutputStream outFileStream = new FileOutputStream(outFile);
		    model.serialize(outFileStream);
		}
}

when i used the output bi file to test it on a string to pull up Fontana from a string the
result returned the whole string....don't know why or what i am doing wrongthe following code
i used to extract Fontana from a stringimport java.io.FileInputStream;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Span;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.SimpleTokenizer;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;

import org.xml.sax.SAXException;


public class CityFinder {

    public String Tokens[];

    public static void main(String[] args) throws IOException, SAXException {

    	CityFinder toi = new CityFinder();
        String cnt;
        cnt="John is planning to specialize in Electrical Engineering in UC Fontana and pursue
a career with IBM.";
        toi.tokenization(cnt);
        String cities = toi.namefind(toi.Tokens);
        String org = toi.orgfind(toi.Tokens);

        System.out.println("City name is : "+cities);
        System.out.println("organization name is: "+org);

    }
        public String namefind(String cnt[]) {
        InputStream is;
        TokenNameFinderModel tnf;
        NameFinderME nf;
        String sd = "";
        try {
            is = new FileInputStream("en-ner-locationNew.bin");
            tnf = new TokenNameFinderModel(is);
            nf = new NameFinderME(tnf);
            Span sp[] = nf.find(cnt);
            String a[] = Span.spansToStrings(sp, cnt);
            StringBuilder fd = new StringBuilder();
            int l = a.length;

            for (int j = 0; j < l; j++) {
                fd = fd.append(a[j] + "\n");

            }
            sd = fd.toString();

        } catch (FileNotFoundException e) {

            e.printStackTrace();
        } catch (InvalidFormatException e) {

            e.printStackTrace();
        } catch (IOException e) {

            e.printStackTrace();
        }
        return sd;
    }

    public String orgfind(String cnt[]) {
        InputStream is;
        TokenNameFinderModel tnf;
        NameFinderME nf;
        String sd = "";
        try {
            is = new FileInputStream("en-ner-organization.bin");
            tnf = new TokenNameFinderModel(is);
            nf = new NameFinderME(tnf);
            Span sp[] = nf.find(cnt);
            String a[] = Span.spansToStrings(sp, cnt);
            StringBuilder fd = new StringBuilder();
            int l = a.length;
            for (int j = 0; j < l; j++) {
                fd = fd.append(a[j] + "\n");

            }

            sd = fd.toString();

        } catch (FileNotFoundException e) {

            e.printStackTrace();
        } catch (InvalidFormatException e) {

            e.printStackTrace();
        } catch (IOException e) {

            e.printStackTrace();
        }
        return sd;

    }
    public void tokenization(String tokens) {

        InputStream is;
        TokenizerModel tm;
        try {
            is = new FileInputStream("en-token.bin");
            tm = new TokenizerModel(is);
            Tokenizer tz = new TokenizerME(tm);
            Tokens = tz.tokenize(tokens);
            // System.out.println(Tokens[1]);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}
can you please let me know where i did wrong...?? 		 	   		  
Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message