Return-Path: X-Original-To: apmail-lucene-java-user-archive@www.apache.org Delivered-To: apmail-lucene-java-user-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5CF43108D6 for ; Tue, 15 Oct 2013 12:14:18 +0000 (UTC) Received: (qmail 90071 invoked by uid 500); 15 Oct 2013 12:14:15 -0000 Delivered-To: apmail-lucene-java-user-archive@lucene.apache.org Received: (qmail 89591 invoked by uid 500); 15 Oct 2013 12:14:07 -0000 Mailing-List: contact java-user-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: java-user@lucene.apache.org Delivered-To: mailing list java-user@lucene.apache.org Received: (qmail 89575 invoked by uid 99); 15 Oct 2013 12:14:06 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Oct 2013 12:14:06 +0000 X-ASF-Spam-Status: No, hits=2.2 required=5.0 tests=HTML_MESSAGE,SPF_PASS X-Spam-Check-By: apache.org Received-SPF: pass (nike.apache.org: domain of kvjudson@hotmail.com designates 65.55.34.92 as permitted sender) Received: from [65.55.34.92] (HELO col0-omc2-s18.col0.hotmail.com) (65.55.34.92) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Oct 2013 12:13:57 +0000 Received: from COL126-W45 ([65.55.34.72]) by col0-omc2-s18.col0.hotmail.com with Microsoft SMTPSVC(6.0.3790.4675); Tue, 15 Oct 2013 05:13:35 -0700 X-TMN: [99Dcck/y94LikX8AN2bplZaG/4fC5Vlx] X-Originating-Email: [kvjudson@hotmail.com] Message-ID: Content-Type: multipart/alternative; boundary="_73320d47-d4e4-496e-b870-cc7e4f3ebbdb_" From: Kyle Judson To: "java-user@lucene.apache.org" Subject: Payloads disabled in 4.5? Date: Tue, 15 Oct 2013 06:13:35 -0600 Importance: Normal MIME-Version: 1.0 X-OriginalArrivalTime: 15 Oct 2013 12:13:35.0891 (UTC) FILETIME=[F93E3630:01CEC99F] X-Virus-Checked: Checked by ClamAV on apache.org --_73320d47-d4e4-496e-b870-cc7e4f3ebbdb_ Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable =0A= =0A= =0A= =0A= =0A= =0A= =0A= =0A= Hi All=2C=0A= =0A= I'm trying to create an index with payloads using Lucene 4.5 but the payloa= ds don't seem to be getting into the index. =0A= =0A= I looked into the code and it looks like storePayloads argument to addOrUpd= ateInternal when called from addOrUpdate in FieldInfos is set to false. Is = this my problem? If so should I be using a different indexer?=0A= =0A= The indexing and reading coded is below. ThanksKyle The indexing code:import java.io.File=3Bimport java.io.IOException=3Bimport= java.nio.ByteBuffer=3B import org.apache.commons.logging.Log=3Bimport org.apache.commons.logging.L= ogFactory=3Bimport org.apache.lucene.analysis.Analyzer=3Bimport org.apache.= lucene.analysis.AnalyzerWrapper=3Bimport org.apache.lucene.analysis.TokenFi= lter=3Bimport org.apache.lucene.analysis.TokenStream=3Bimport org.apache.lu= cene.analysis.standard.StandardAnalyzer=3Bimport org.apache.lucene.analysis= .tokenattributes.PayloadAttribute=3Bimport org.apache.lucene.document.Docum= ent=3Bimport org.apache.lucene.document.Field=3Bimport org.apache.lucene.do= cument.FieldType=3Bimport org.apache.lucene.document.TextField=3Bimport org= .apache.lucene.index.IndexWriter=3Bimport org.apache.lucene.index.IndexWrit= erConfig=3Bimport org.apache.lucene.index.IndexWriterConfig.OpenMode=3Bimpo= rt org.apache.lucene.index.IndexableField=3Bimport org.apache.lucene.store.= Directory=3Bimport org.apache.lucene.store.FSDirectory=3Bimport org.apache.= lucene.util.BytesRef=3Bimport org.apache.lucene.util.Version=3B public class TestIndex2 { private static Log LOG =3D LogFactory.getLog(Test= Index2.class)=3B MetricAnalyzer analyzer=3B Directory directory=3B IndexWr= iterConfig configIndex=3B IndexWriter iwriter=3B public TestIndex2(File fi= leOut) throws IOException { analyzer =3D new MetricAnalyzer(Analyzer.PER_F= IELD_REUSE_STRATEGY)=3B directory =3D FSDirectory.open(fileOut)=3B conf= igIndex =3D new IndexWriterConfig(Version.LUCENE_45=2C analyzer)=3B config= Index.setOpenMode(OpenMode.CREATE)=3B iwriter =3D new IndexWriter(director= y=2C configIndex)=3B } // end TestIndex2 public void close() { if (iwriter !=3D null) { try { iwriter.close()= =3B } catch (IOException e) { LOG.error("Unable to close IndexWriter"= =2C e)=3B } } if (directory !=3D null) { try { directory.close()= =3B } catch (IOException e) { LOG.error("Unable to close Directory"=2C= e)=3B } } } // end close BytesRef encodeMetrics(long[] metrics) { Byt= eBuffer buffer =3D ByteBuffer.allocate(metrics.length * 8)=3B for (long me= tric : metrics) { buffer.putLong(metric)=3B } return new BytesRef(buffe= r.array())=3B } // end encodeMetrics public void index() throws IOExceptio= n { FieldType type =3D new FieldType(TextField.TYPE_NOT_STORED)=3B type.s= etStoreTermVectorPayloads(true)=3B type.setStoreTermVectorPositions(true)= =3B type.setStoreTermVectors(true)=3B Document doc =3D new Document()= =3B long[] metrics =3D { 1=2C 2=2C 3 }=3B analyzer.setPayload(encodeMetric= s(metrics))=3B IndexableField indexField =3D new Field("test"=2C "one"= =2C type)=3B doc.add(indexField)=3B indexField =3D new Field("test"=2C "t= wo"=2C type)=3B doc.add(indexField)=3B indexField =3D new Field("test_query"=2C "one"=2C type)=3B doc.add(index= Field)=3B indexField =3D new Field("test_query"=2C "two"=2C type)=3B doc.= add(indexField)=3B iwriter.addDocument(doc)=3B } // end index public cl= ass MetricTokenFilter extends TokenFilter { private MetricAnalyzer analy= zer=3B private PayloadAttribute payloadAttr=3B protected MetricTokenFilter(MetricAnalyzer analyzer=2C TokenStream input)= { super(input)=3B this.analyzer =3D analyzer=3B this.payloadAttr =3D= addAttribute(PayloadAttribute.class)=3B } // end MetricTokenFilter @Ov= erride public boolean incrementToken() throws IOException { payloadAttr.= setPayload(analyzer.getPayload())=3B return input.incrementToken()=3B } = // end incrementToken } // end class MetricTokenFilter public class Metr= icAnalyzer extends AnalyzerWrapper { BytesRef payload=3B protected Me= tricAnalyzer(Analyzer.ReuseStrategy reuseStrategy) { super(reuseStrategy)= =3B } // end MetricAnalyzer protected void setPayload(BytesRef payload)= { this.payload =3D payload=3B } protected BytesRef getPayload() { return = payload=3B } @Override protected Analyzer getWrappedAnalyzer(String fieldName) { re= turn new StandardAnalyzer(Version.LUCENE_45)=3B } // end getWrappedAnalyze= r @Override protected TokenStreamComponents wrapComponents(String field= Name=2C TokenStreamComponents components) { if (fieldName.endsWith("_quer= y")) { return components=3B } else { MetricTokenFilter filter =3D n= ew MetricTokenFilter(this=2C components.getTokenStream())=3B return new = TokenStreamComponents(components.getTokenizer()=2C filter)=3B } } // end= wrapComponents } // end class MetricAnalyzer public static void main(String[] args) thro= ws IOException { File fileOut =3D new File("test_index")=3B TestIndex2 iw= riter =3D new TestIndex2(fileOut)=3B try { iwriter.index()=3B } finally= { iwriter.close()=3B } } // end main } // end class TestIndex2 The reading code:import java.io.File=3Bimport java.io.IOException=3Bimport = java.nio.ByteBuffer=3Bimport java.util.List=3B import org.apache.lucene.index.AtomicReaderContext=3Bimport org.apache.luce= ne.index.DirectoryReader=3Bimport org.apache.lucene.index.DocsAndPositionsE= num=3Bimport org.apache.lucene.index.IndexReader=3Bimport org.apache.lucene= .index.Terms=3Bimport org.apache.lucene.index.TermsEnum=3Bimport org.apache= .lucene.search.DocIdSetIterator=3Bimport org.apache.lucene.store.Directory= =3Bimport org.apache.lucene.store.FSDirectory=3Bimport org.apache.lucene.ut= il.Bits=3Bimport org.apache.lucene.util.BytesRef=3B public class TestQuery2 { public static long[] decodeMetrics(BytesRef metrics=2C int count=2C long[]= reuse) { if (reuse =3D=3D null) { reuse =3D new long[count]=3B } By= teBuffer buffer =3D ByteBuffer.wrap(metrics.bytes)=3B for (int x=3D0=3B x<= count=3B x++) { reuse[x] =3D buffer.getLong()=3B } return reuse=3B } //= end decodeMetrics public static void report (IndexReader ireader=2C Strin= g field) throws IOException { int count =3D 3=3B List leaves =3D ireader.leaves()=3B for (AtomicReaderContext leaf : leaves)= { Terms t =3D leaf.reader().terms(field)=3B if (t =3D=3D null) { Sy= stem.out.println("Invalid field: '"+field+"'")=3B return=3B } TermsE= num terms =3D t.iterator(null)=3B BytesRef term=3B long[] metrics =3D n= ull=3B while ((term =3D terms.next()) !=3D null) { int docCount =3D te= rms.docFreq()=3B Bits bits =3D new Bits.MatchAllBits(docCount)=3B System.out.print ("\t" + term.utf8ToString() + "\t")=3B boolean first =3D true=3B DocsAndPositionsEnum docPositions =3D term= s.docsAndPositions(bits=2C null=2C DocsAndPositionsEnum.FLAG_PAYLOADS)=3B = if (docPositions =3D=3D null) { System.out.println("No doc positions"= )=3B } else { while (docPositions.nextDoc() !=3D DocIdSetIterator.NO= _MORE_DOCS) { for (int pos=3D0=3B pos