Return-Path: X-Original-To: apmail-lucene-java-user-archive@www.apache.org Delivered-To: apmail-lucene-java-user-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 43F9510B21 for ; Tue, 15 Oct 2013 13:25:13 +0000 (UTC) Received: (qmail 45413 invoked by uid 500); 15 Oct 2013 13:25:07 -0000 Delivered-To: apmail-lucene-java-user-archive@lucene.apache.org Received: (qmail 45046 invoked by uid 500); 15 Oct 2013 13:25:06 -0000 Mailing-List: contact java-user-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-user@lucene.apache.org Delivered-To: mailing list java-user@lucene.apache.org Received: (qmail 44877 invoked by uid 99); 15 Oct 2013 13:25:05 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Oct 2013 13:25:05 +0000 X-ASF-Spam-Status: No, hits=-0.7 required=5.0 tests=RCVD_IN_DNSWL_LOW,SPF_PASS X-Spam-Check-By: apache.org Received-SPF: pass (nike.apache.org: local policy includes SPF record at spf.trusted-forwarder.org) Received: from [209.85.128.181] (HELO mail-ve0-f181.google.com) (209.85.128.181) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Oct 2013 13:24:59 +0000 Received: by mail-ve0-f181.google.com with SMTP id pa12so632813veb.40 for ; Tue, 15 Oct 2013 06:24:38 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:content-type:content-transfer-encoding; bh=1FlbAxuaeVmLhIde/cdBRFfWotiNG3nsbeWzasOf6nc=; b=TR1Unj+SIuwVY6IZe2z3I3yUIeG2QzbvZJE+cyRWSX4E/Rz9W/ZFboxR2Q9s/jLFYO O/nVIlcytw8EmSH8rmkNg2yddq2rHf05CCKQI95R/+pi0kvDAK1PJR0P2C7opKNJAdaj Q6yxYBrgoH0hpuhWCksQOHouA6VqdFbR0QMVeQ4mit6R0I+g0/dqM/cNINHDMhitLti1 ak88D0GgoKLvxeV327H6SIZwk9hA5XFcg9xwmffLWEwPvT/mr8WfoDhV2pEyU1MMnpl6 MRugbYInqeRVWkcXAaN3XYB2E4/qvIDUDph8IDLHlFYg5+VW4NsXYsRMavA52YKNljle NoFA== X-Gm-Message-State: ALoCoQmmp3uooZkqd+ovZLeun5liiOdhGC9YGroDNqu2aWq3EnfA4pusymqW6dISxB10+NaEaD/e X-Received: by 10.58.44.197 with SMTP id g5mr11791218vem.0.1381843478073; Tue, 15 Oct 2013 06:24:38 -0700 (PDT) MIME-Version: 1.0 Received: by 10.220.72.131 with HTTP; Tue, 15 Oct 2013 06:24:17 -0700 (PDT) In-Reply-To: References: From: Michael McCandless Date: Tue, 15 Oct 2013 09:24:17 -0400 Message-ID: Subject: Re: Payloads disabled in 4.5? To: Lucene Users Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable X-Virus-Checked: Checked by ClamAV on apache.org Something catastrophic went wrong with the whitespace in your email ... But I think the problem is in your incrementToken impl: you cannot set the payload, and then call input.incrementToken, because input.incrementToken will clear all attributes (including payload). Try reversing the order of those two? Mike McCandless http://blog.mikemccandless.com On Tue, Oct 15, 2013 at 8:13 AM, Kyle Judson wrote: > > > > > > > > > Hi All, > > > I'm trying to create an index with payloads using Lucene 4.5 but the payl= oads don't seem to be getting into the index. > > > I looked into the code and it looks like storePayloads argument to addOrU= pdateInternal when called from addOrUpdate in FieldInfos is set to false. I= s this my problem? If so should I be using a different indexer? > > > The indexing and reading coded is below. > ThanksKyle > The indexing code:import java.io.File;import java.io.IOException;import j= ava.nio.ByteBuffer; > import org.apache.commons.logging.Log;import org.apache.commons.logging.L= ogFactory;import org.apache.lucene.analysis.Analyzer;import org.apache.luce= ne.analysis.AnalyzerWrapper;import org.apache.lucene.analysis.TokenFilter;i= mport org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analy= sis.standard.StandardAnalyzer;import org.apache.lucene.analysis.tokenattrib= utes.PayloadAttribute;import org.apache.lucene.document.Document;import org= .apache.lucene.document.Field;import org.apache.lucene.document.FieldType;i= mport org.apache.lucene.document.TextField;import org.apache.lucene.index.I= ndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apac= he.lucene.index.IndexWriterConfig.OpenMode;import org.apache.lucene.index.I= ndexableField;import org.apache.lucene.store.Directory;import org.apache.lu= cene.store.FSDirectory;import org.apache.lucene.util.BytesRef;import org.ap= ache.lucene.util.Version; > > public class TestIndex2 { private static Log LOG =3D LogFactory.get= Log(TestIndex2.class); MetricAnalyzer analyzer; Directory = directory; IndexWriterConfig configIndex; IndexWriter iwriter; = public TestIndex2(File fileOut) throws IOException { analyzer= =3D new MetricAnalyzer(Analyzer.PER_FIELD_REUSE_STRATEGY); = directory =3D FSDirectory.open(fileOut); configInde= x =3D new IndexWriterConfig(Version.LUCENE_45, analyzer); con= figIndex.setOpenMode(OpenMode.CREATE); iwriter =3D new IndexW= riter(directory, configIndex); } // end TestIndex2 > public void close() { if (iwriter !=3D null) { = try { iwriter.close(); = } catch (IOException e) { LOG.error("Un= able to close IndexWriter", e); } } = if (directory !=3D null) { try { = directory.close(); } catch (IOException= e) { LOG.error("Unable to close Directory", = e); } } } // end close By= tesRef encodeMetrics(long[] metrics) { ByteBuffer buffer =3D= ByteBuffer.allocate(metrics.length * 8); for (long metric : met= rics) { buffer.putLong(metric); } r= eturn new BytesRef(buffer.array()); } // end encodeMetrics publ= ic void index() throws IOException { FieldType type =3D new = FieldType(TextField.TYPE_NOT_STORED); type.setStoreTermVectorP= ayloads(true); type.setStoreTermVectorPositions(true); typ= e.setStoreTermVectors(true); Document doc =3D new D= ocument(); > long[] metrics =3D { 1, 2, 3 }; = analyzer.setPayload(encodeMetrics(metrics)); In= dexableField indexField =3D new Field("test", "one", type); doc= .add(indexField); indexField =3D new Field("test", "two", type);= doc.add(indexField); > indexField =3D new Field("test_query", "one", type); = doc.add(indexField); indexField =3D new Field("test_que= ry", "two", type); doc.add(indexField); = iwriter.addDocument(doc); } // end index public class M= etricTokenFilter extends TokenFilter { private M= etricAnalyzer analyzer; private PayloadAttribute payloadAttr= ; > protected MetricTokenFilter(MetricAnalyzer analyzer, Toke= nStream input) { super(input); this= .analyzer =3D analyzer; this.payloadAttr =3D addAttri= bute(PayloadAttribute.class); } // end MetricTokenFilter = @Override public boolean incrementT= oken() throws IOException { payloadAttr.setPayload(analy= zer.getPayload()); return input.incrementToken(); = } // end incrementToken } // end class MetricTokenFilter = public class MetricAnalyzer extends AnalyzerWrapper { = BytesRef payload; protecte= d MetricAnalyzer(Analyzer.ReuseStrategy reuseStrategy) { = super(reuseStrategy); } // end MetricAnalyzer = protected void setPayload(BytesRef payload) { this.payload =3D pa= yload; } protected BytesRef getPayload() { return payload; } > @Override protected Analyzer getWrappedAnal= yzer(String fieldName) { return new StandardAnalyzer(= Version.LUCENE_45); } // end getWrappedAnalyzer = @Override protected TokenStreamComponents wrapCompon= ents(String fieldName, TokenStreamComponents components) { = if (fieldName.endsWith("_query")) { return c= omponents; } else { Met= ricTokenFilter filter =3D new MetricTokenFilter(this, components.getTokenSt= ream()); return new TokenStreamComponents(compon= ents.getTokenizer(), filter); } } // end w= rapComponents > } // end class MetricAnalyzer public static void main(S= tring[] args) throws IOException { File fileOut =3D new File("t= est_index"); TestIndex2 iwriter =3D new TestIndex2(fileOut); = try { iwriter.index(); } finally { = iwriter.close(); } } // end main > } // end class TestIndex2 > > The reading code:import java.io.File;import java.io.IOException;import ja= va.nio.ByteBuffer;import java.util.List; > import org.apache.lucene.index.AtomicReaderContext;import org.apache.luce= ne.index.DirectoryReader;import org.apache.lucene.index.DocsAndPositionsEnu= m;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index= .Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.se= arch.DocIdSetIterator;import org.apache.lucene.store.Directory;import org.a= pache.lucene.store.FSDirectory;import org.apache.lucene.util.Bits;import or= g.apache.lucene.util.BytesRef; > > public class TestQuery2 { > public static long[] decodeMetrics(BytesRef metrics, int count, l= ong[] reuse) { if (reuse =3D=3D null) { reuse = =3D new long[count]; } ByteBuf= fer buffer =3D ByteBuffer.wrap(metrics.bytes); for (int x=3D0; = x leaves =3D ireader.leaves(); for (AtomicReaderContext leaf := leaves) { Terms t =3D leaf.reader().terms(field); = if (t =3D=3D null) { System.= out.println("Invalid field: '"+field+"'"); re= turn; } TermsEnum terms =3D t.iterato= r(null); BytesRef term; long[] metrics= =3D null; while ((term =3D terms.next()) !=3D null) { = int docCount =3D terms.docFreq(); = Bits bits =3D new Bits.MatchAllBits(docCount); > System.out.print ("\t" + term.utf8ToStrin= g() + "\t"); > boolean first =3D true; = DocsAndPositionsEnum docPositions =3D terms.docsAndPositions(bits,= null, DocsAndPositionsEnum.FLAG_PAYLOADS); if = (docPositions =3D=3D null) { System.out= .println("No doc positions"); } else { = while (docPositions.nextDoc() !=3D DocIdSetItera= tor.NO_MORE_DOCS) { for (int = pos=3D0; pos DirectoryReader ireader =3D DirectoryReader.open(director= y); try { TestQuery2.report(ireader, "test")= ; } finally { directory.close(); = } > } // end main } // end class TestQuery2 > --------------------------------------------------------------------- To unsubscribe, e-mail: java-user-unsubscribe@lucene.apache.org For additional commands, e-mail: java-user-help@lucene.apache.org