Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id EF4FC200B92 for ; Wed, 14 Sep 2016 00:44:01 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id EDF39160ADA; Tue, 13 Sep 2016 22:44:01 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A854D160AD8 for ; Wed, 14 Sep 2016 00:43:59 +0200 (CEST) Received: (qmail 28766 invoked by uid 500); 13 Sep 2016 22:43:58 -0000 Mailing-List: contact commits-help@geode.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.incubator.apache.org Delivered-To: mailing list commits@geode.incubator.apache.org Received: (qmail 28757 invoked by uid 99); 13 Sep 2016 22:43:58 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Sep 2016 22:43:58 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 4FC32CEF98 for ; Tue, 13 Sep 2016 22:43:58 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.344 X-Spam-Level: X-Spam-Status: No, score=-4.344 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.124] autolearn=disabled Received: from mx2-lw-us.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id 6qJlJqtui1Ia for ; Tue, 13 Sep 2016 22:43:53 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-us.apache.org (ASF Mail Server at mx2-lw-us.apache.org) with SMTP id D14F95FC23 for ; Tue, 13 Sep 2016 22:43:51 +0000 (UTC) Received: (qmail 28385 invoked by uid 99); 13 Sep 2016 22:43:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Sep 2016 22:43:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 47D8AE07F4; Tue, 13 Sep 2016 22:43:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hiteshkhamesra@apache.org To: commits@geode.incubator.apache.org Date: Tue, 13 Sep 2016 22:44:35 -0000 Message-Id: <6e84623611b54364ae6e7797ecd7a49c@git.apache.org> In-Reply-To: <69ace8383aae47d1b0c2dc7793e01b2a@git.apache.org> References: <69ace8383aae47d1b0c2dc7793e01b2a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [47/61] [abbrv] incubator-geode git commit: GEODE-37 change package name from com.gemstone.gemfire (for ./geode-lucene/src/main/java/com/gemstone/gemfire)to org.apache.geode for(to ./geode-lucene/src/main/java/org/apache/geode) archived-at: Tue, 13 Sep 2016 22:44:02 -0000 http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexFactory.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexFactory.java new file mode 100755 index 0000000..b6ac867 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexFactory.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; + +public class LuceneIndexFactory { + public LuceneIndexFactory() { + } + + public LuceneIndexImpl create(String indexName, String regionPath, GemFireCacheImpl cache) { + return new LuceneIndexForPartitionedRegion(indexName, regionPath, cache); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java new file mode 100644 index 0000000..b64e026 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexForPartitionedRegion.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.cache.AttributesFactory; +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.DataPolicy; +import com.gemstone.gemfire.cache.PartitionAttributes; +import com.gemstone.gemfire.cache.PartitionAttributesFactory; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.RegionAttributes; +import com.gemstone.gemfire.cache.RegionShortcut; +import com.gemstone.gemfire.cache.asyncqueue.AsyncEventQueue; +import com.gemstone.gemfire.cache.asyncqueue.internal.AsyncEventQueueFactoryImpl; +import com.gemstone.gemfire.cache.execute.FunctionService; +import com.gemstone.gemfire.cache.execute.ResultCollector; +import com.gemstone.gemfire.cache.lucene.internal.directory.DumpDirectoryFiles; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.ChunkKey; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.File; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystemStats; +import com.gemstone.gemfire.cache.lucene.internal.repository.RepositoryManager; +import com.gemstone.gemfire.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer; +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; + +/* wrapper of IndexWriter */ +public class LuceneIndexForPartitionedRegion extends LuceneIndexImpl { + protected Region fileRegion; + protected Region chunkRegion; + protected final FileSystemStats fileSystemStats; + + public LuceneIndexForPartitionedRegion(String indexName, String regionPath, Cache cache) { + super(indexName, regionPath, cache); + + final String statsName = indexName + "-" + regionPath; + this.fileSystemStats = new FileSystemStats(cache.getDistributedSystem(), statsName); + } + + protected RepositoryManager createRepositoryManager() { + RegionShortcut regionShortCut; + final boolean withPersistence = withPersistence(); + RegionAttributes regionAttributes = dataRegion.getAttributes(); + final boolean withStorage = regionAttributes.getPartitionAttributes().getLocalMaxMemory()>0; + + // TODO: 1) dataRegion should be withStorage + // 2) Persistence to Persistence + // 3) Replicate to Replicate, Partition To Partition + // 4) Offheap to Offheap + if (!withStorage) { + throw new IllegalStateException("The data region to create lucene index should be with storage"); + } + if (withPersistence) { + // TODO: add PartitionedRegionAttributes instead + regionShortCut = RegionShortcut.PARTITION_PERSISTENT; + } else { + regionShortCut = RegionShortcut.PARTITION; + } + + // create PR fileRegion, but not to create its buckets for now + final String fileRegionName = createFileRegionName(); + PartitionAttributes partitionAttributes = dataRegion.getPartitionAttributes(); + if (!fileRegionExists(fileRegionName)) { + fileRegion = createFileRegion(regionShortCut, fileRegionName, partitionAttributes, regionAttributes); + } + + // create PR chunkRegion, but not to create its buckets for now + final String chunkRegionName = createChunkRegionName(); + if (!chunkRegionExists(chunkRegionName)) { + chunkRegion = createChunkRegion(regionShortCut, fileRegionName, partitionAttributes, chunkRegionName, regionAttributes); + } + fileSystemStats.setFileSupplier(() -> (int) getFileRegion().getLocalSize()); + fileSystemStats.setChunkSupplier(() -> (int) getChunkRegion().getLocalSize()); + fileSystemStats.setBytesSupplier(() -> getChunkRegion().getPrStats().getDataStoreBytesInUse()); + + // we will create RegionDirectories on the fly when data comes in + HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames()); + return new PartitionedRepositoryManager(this, mapper); + } + + public PartitionedRegion getFileRegion() { + return (PartitionedRegion) fileRegion; + } + + public PartitionedRegion getChunkRegion() { + return (PartitionedRegion) chunkRegion; + } + + public FileSystemStats getFileSystemStats() { + return fileSystemStats; + } + + boolean fileRegionExists(String fileRegionName) { + return cache. getRegion(fileRegionName) != null; + } + + Region createFileRegion(final RegionShortcut regionShortCut, + final String fileRegionName, + final PartitionAttributes partitionAttributes, + final RegionAttributes regionAttributes) { + return createRegion(fileRegionName, regionShortCut, this.regionPath, partitionAttributes, regionAttributes); + } + + public String createFileRegionName() { + return LuceneServiceImpl.getUniqueIndexName(indexName, regionPath)+".files"; + } + + boolean chunkRegionExists(String chunkRegionName) { + return cache. getRegion(chunkRegionName) != null; + } + + Region createChunkRegion(final RegionShortcut regionShortCut, + final String fileRegionName, + final PartitionAttributes partitionAttributes, final String chunkRegionName, final RegionAttributes regionAttributes) { + return createRegion(chunkRegionName, regionShortCut, fileRegionName, partitionAttributes, regionAttributes); + } + + public String createChunkRegionName() { + return LuceneServiceImpl.getUniqueIndexName(indexName, regionPath) + ".chunks"; + } + + private PartitionAttributesFactory configureLuceneRegionAttributesFactory(PartitionAttributesFactory attributesFactory, PartitionAttributes dataRegionAttributes) { + attributesFactory.setTotalNumBuckets(dataRegionAttributes.getTotalNumBuckets()); + attributesFactory.setRedundantCopies(dataRegionAttributes.getRedundantCopies()); + return attributesFactory; + } + + protected Region createRegion(final String regionName, + final RegionShortcut regionShortCut, + final String colocatedWithRegionName, + final PartitionAttributes partitionAttributes, + final RegionAttributes regionAttributes) + { + PartitionAttributesFactory partitionAttributesFactory = new PartitionAttributesFactory(); + partitionAttributesFactory.setColocatedWith(colocatedWithRegionName); + configureLuceneRegionAttributesFactory(partitionAttributesFactory, partitionAttributes); + + // Create AttributesFactory based on input RegionShortcut + RegionAttributes baseAttributes = this.cache.getRegionAttributes(regionShortCut.toString()); + AttributesFactory factory = new AttributesFactory(baseAttributes); + factory.setPartitionAttributes(partitionAttributesFactory.create()); + factory.setDiskStoreName(regionAttributes.getDiskStoreName()); + RegionAttributes attributes = factory.create(); + + return createRegion(regionName, attributes); + } + + public void close() { + // TODO Auto-generated method stub + + } + + @Override + public void dumpFiles(final String directory) { + ResultCollector results = FunctionService.onRegion(getDataRegion()) + .withArgs(new String[] {directory, indexName}) + .execute(DumpDirectoryFiles.ID); + results.getResult(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java new file mode 100644 index 0000000..67461a9 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +import com.gemstone.gemfire.InternalGemFireError; +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.DataPolicy; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.RegionAttributes; +import com.gemstone.gemfire.cache.asyncqueue.AsyncEventQueue; +import com.gemstone.gemfire.cache.asyncqueue.internal.AsyncEventQueueFactoryImpl; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.ChunkKey; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.File; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystemStats; +import com.gemstone.gemfire.cache.lucene.internal.repository.RepositoryManager; +import com.gemstone.gemfire.cache.lucene.internal.xml.LuceneIndexCreation; +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; +import com.gemstone.gemfire.internal.cache.InternalRegionArguments; +import com.gemstone.gemfire.internal.cache.LocalRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; +import com.gemstone.gemfire.internal.i18n.LocalizedStrings; +import com.gemstone.gemfire.internal.logging.LogService; + +public abstract class LuceneIndexImpl implements InternalLuceneIndex { + protected static final Logger logger = LogService.getLogger(); + + protected final String indexName; + protected final String regionPath; + protected final Cache cache; + protected final LuceneIndexStats indexStats; + + protected boolean hasInitialized = false; + protected Map fieldAnalyzers; + protected String[] searchableFieldNames; + protected RepositoryManager repositoryManager; + protected Analyzer analyzer; + protected LocalRegion dataRegion; + + protected LuceneIndexImpl(String indexName, String regionPath, Cache cache) { + this.indexName = indexName; + this.regionPath = regionPath; + this.cache = cache; + + final String statsName = indexName + "-" + regionPath; + this.indexStats = new LuceneIndexStats(cache.getDistributedSystem(), statsName); + } + + @Override + public String getName() { + return this.indexName; + } + + @Override + public String getRegionPath() { + return this.regionPath; + } + + protected LocalRegion getDataRegion() { + return (LocalRegion)cache.getRegion(regionPath); + } + + protected boolean withPersistence() { + RegionAttributes ra = dataRegion.getAttributes(); + DataPolicy dp = ra.getDataPolicy(); + final boolean withPersistence = dp.withPersistence(); + return withPersistence; + } + + protected void setSearchableFields(String[] fields) { + searchableFieldNames = fields; + } + + @Override + public boolean waitUntilFlushed(int maxWaitInMillisecond) { + String aeqId = LuceneServiceImpl.getUniqueIndexName(indexName, regionPath); + AsyncEventQueue queue = (AsyncEventQueue)cache.getAsyncEventQueue(aeqId); + boolean flushed = false; + if (queue != null) { + long start = System.nanoTime(); + while (System.nanoTime() - start < TimeUnit.MILLISECONDS.toNanos(maxWaitInMillisecond)) { + if (0 == queue.size()) { + flushed = true; + break; + } else { + try { + Thread.sleep(200); + } catch (InterruptedException e) { + } + } + } + } else { + throw new IllegalArgumentException("The AEQ does not exist for the index "+indexName+" region "+regionPath); + } + + return flushed; + } + + @Override + public String[] getFieldNames() { + return searchableFieldNames; + } + + @Override + public Map getFieldAnalyzers() { + return this.fieldAnalyzers; + } + + public RepositoryManager getRepositoryManager() { + return this.repositoryManager; + } + + public void setAnalyzer(Analyzer analyzer) { + if (analyzer == null) { + this.analyzer = new StandardAnalyzer(); + } else { + this.analyzer = analyzer; + } + } + + public Analyzer getAnalyzer() { + return this.analyzer; + } + + public Cache getCache() { + return this.cache; + } + + public void setFieldAnalyzers(Map fieldAnalyzers) { + this.fieldAnalyzers = fieldAnalyzers == null ? null : Collections.unmodifiableMap(fieldAnalyzers); + } + + public LuceneIndexStats getIndexStats() { + return indexStats; + } + + protected void initialize() { + if (!hasInitialized) { + /* create index region */ + dataRegion = getDataRegion(); + //assert dataRegion != null; + + repositoryManager = createRepositoryManager(); + + // create AEQ, AEQ listener and specify the listener to repositoryManager + createAEQ(dataRegion); + + addExtension(dataRegion); + hasInitialized = true; + } + } + + protected abstract RepositoryManager createRepositoryManager(); + + protected AsyncEventQueue createAEQ(Region dataRegion) { + return createAEQ(createAEQFactory(dataRegion)); + } + + private AsyncEventQueueFactoryImpl createAEQFactory(final Region dataRegion) { + AsyncEventQueueFactoryImpl factory = (AsyncEventQueueFactoryImpl) cache.createAsyncEventQueueFactory(); + if (dataRegion instanceof PartitionedRegion) { + factory.setParallel(true); // parallel AEQ for PR + } else { + factory.setParallel(false); // TODO: not sure if serial AEQ working or not + } + factory.setMaximumQueueMemory(1000); + factory.setDispatcherThreads(10); + factory.setIsMetaQueue(true); + if (dataRegion.getAttributes().getDataPolicy().withPersistence()) { + factory.setPersistent(true); + } + factory.setDiskStoreName(dataRegion.getAttributes().getDiskStoreName()); + factory.setDiskSynchronous(dataRegion.getAttributes().isDiskSynchronous()); + factory.setForwardExpirationDestroy(true); + return factory; + } + + private AsyncEventQueue createAEQ(AsyncEventQueueFactoryImpl factory) { + LuceneEventListener listener = new LuceneEventListener(repositoryManager); + String aeqId = LuceneServiceImpl.getUniqueIndexName(getName(), regionPath); + AsyncEventQueue indexQueue = factory.create(aeqId, listener); + return indexQueue; + } + +/** + * Register an extension with the region + * so that xml will be generated for this index. + */ + protected void addExtension(LocalRegion dataRegion) { + LuceneIndexCreation creation = new LuceneIndexCreation(); + creation.setName(this.getName()); + creation.addFieldNames(this.getFieldNames()); + creation.setRegion(dataRegion); + creation.setFieldAnalyzers(this.getFieldAnalyzers()); + dataRegion.getExtensionPoint().addExtension(creation); + } + + protected Region createRegion(final String regionName, final RegionAttributes attributes) { + // Create InternalRegionArguments to set isUsedForMetaRegion true to suppress xml generation (among other things) + InternalRegionArguments ira = new InternalRegionArguments().setDestroyLockFlag(true).setRecreateFlag(false) + .setSnapshotInputStream(null).setImageTarget(null).setIsUsedForMetaRegion(true); + + // Create the region + try { + return ((GemFireCacheImpl)this.cache).createVMRegion(regionName, attributes, ira); + } catch (Exception e) { + InternalGemFireError ige = new InternalGemFireError(LocalizedStrings.GemFireCache_UNEXPECTED_EXCEPTION.toLocalizedString()); + ige.initCause(e); + throw ige; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexStats.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexStats.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexStats.java new file mode 100644 index 0000000..220ae0e --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexStats.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import static com.gemstone.gemfire.distributed.internal.DistributionStats.getStatTime; + +import java.util.function.IntSupplier; + +import com.gemstone.gemfire.StatisticDescriptor; +import com.gemstone.gemfire.Statistics; +import com.gemstone.gemfire.StatisticsFactory; +import com.gemstone.gemfire.StatisticsType; +import com.gemstone.gemfire.StatisticsTypeFactory; +import com.gemstone.gemfire.internal.CopyOnWriteHashSet; +import com.gemstone.gemfire.internal.statistics.StatisticsTypeFactoryImpl; + +public class LuceneIndexStats { + // statistics type + private static final StatisticsType statsType; + private static final String statsTypeName = "LuceneIndexStats"; + private static final String statsTypeDescription = "Statistics about lucene indexes"; + + private static final int queryExecutionsId; + private static final int queryExecutionTimeId; + private static final int queryExecutionsInProgressId; + private static final int queryExecutionTotalHitsId; + private static final int updatesId; + private static final int updateTimeId; + private static final int updatesInProgressId; + private static final int commitsId; + private static final int commitTimeId; + private static final int commitsInProgressId; + private static final int documentsId; + + private final Statistics stats; + private final CopyOnWriteHashSet documentsSuppliers = new CopyOnWriteHashSet<>(); + + static { + final StatisticsTypeFactory f = StatisticsTypeFactoryImpl.singleton(); + statsType = f.createType( + statsTypeName, + statsTypeDescription, + new StatisticDescriptor[] { + f.createIntCounter("queryExecutions", "Number of lucene queries executed on this member", "operations"), + f.createLongCounter("queryExecutionTime", "Amount of time spent executing lucene queries", "nanoseconds"), + f.createIntGauge("queryExecutionsInProgress", "Number of query executions currently in progress", "operations"), + f.createLongCounter("queryExecutionTotalHits", "Total number of documents returned by query executions", "entries"), + f.createIntCounter("updates", "Number of lucene index documents added/removed on this member", "operations"), + f.createLongCounter("updateTime", "Amount of time spent adding or removing documents from the index", "nanoseconds"), + f.createIntGauge("updatesInProgress", "Number of index updates in progress", "operations"), + f.createIntCounter("commits", "Number of lucene index commits on this member", "operations"), + f.createLongCounter("commitTime", "Amount of time spent in lucene index commits", "nanoseconds"), + f.createIntGauge("commitsInProgress", "Number of lucene index commits in progress", "operations"), + f.createIntGauge("documents", "Number of documents in the index", "documents"), + } + ); + + queryExecutionsId = statsType.nameToId("queryExecutions"); + queryExecutionTimeId = statsType.nameToId("queryExecutionTime"); + queryExecutionsInProgressId = statsType.nameToId("queryExecutionsInProgress"); + queryExecutionTotalHitsId = statsType.nameToId("queryExecutionTotalHits"); + updatesId = statsType.nameToId("updates"); + updateTimeId = statsType.nameToId("updateTime"); + updatesInProgressId = statsType.nameToId("updatesInProgress"); + commitsId = statsType.nameToId("commits"); + commitTimeId = statsType.nameToId("commitTime"); + commitsInProgressId = statsType.nameToId("commitsInProgress"); + documentsId = statsType.nameToId("documents"); + } + + public LuceneIndexStats(StatisticsFactory f, String name) { + this.stats = f.createAtomicStatistics(statsType, name); + stats.setIntSupplier(documentsId, this::computeDocumentCount); + } + + /** + * @return the timestamp that marks the start of the operation + */ + public long startQuery() { + stats.incInt(queryExecutionsInProgressId, 1); + return getStatTime(); + } + /** + * @param start the timestamp taken when the operation started + */ + public void endQuery(long start, final int totalHits) { + stats.incLong(queryExecutionTimeId, getStatTime()-start); + stats.incInt(queryExecutionsInProgressId, -1); + stats.incInt(queryExecutionsId, 1); + stats.incLong(queryExecutionTotalHitsId, totalHits); + } + + /** + * @return the timestamp that marks the start of the operation + */ + public long startUpdate() { + stats.incInt(updatesInProgressId, 1); + return getStatTime(); + } + /** + * @param start the timestamp taken when the operation started + */ + public void endUpdate(long start) { + stats.incLong(updateTimeId, getStatTime()-start); + stats.incInt(updatesInProgressId, -1); + stats.incInt(updatesId, 1); + } + + /** + * @return the timestamp that marks the start of the operation + */ + public long startCommit() { + stats.incInt(commitsInProgressId, 1); + return getStatTime(); + } + /** + * @param start the timestamp taken when the operation started + */ + public void endCommit(long start) { + stats.incLong(commitTimeId, getStatTime()-start); + stats.incInt(commitsInProgressId, -1); + stats.incInt(commitsId, 1); + } + + public void addDocumentsSupplier(IntSupplier supplier) { + this.documentsSuppliers.add(supplier); + } + + public void removeDocumentsSupplier(IntSupplier supplier) { + this.documentsSuppliers.remove(supplier); + } + + public int getDocuments() { + return this.stats.getInt(documentsId); + } + + private int computeDocumentCount() { + return this.documentsSuppliers.stream() + .mapToInt(IntSupplier::getAsInt) + .sum(); + } + + public int getQueryExecutions() { + return stats.getInt(queryExecutionsId); + } + + public long getQueryExecutionTime() { + return stats.getLong(queryExecutionTimeId); + } + + public int getQueryExecutionsInProgress() { + return stats.getInt(queryExecutionsInProgressId); + } + + public long getQueryExecutionTotalHits() { + return stats.getLong(queryExecutionTotalHitsId); + } + + public int getUpdates() { + return stats.getInt(updatesId); + } + + public long getUpdateTime() { + return stats.getLong(updateTimeId); + } + + public int getUpdatesInProgress() { + return stats.getInt(updatesInProgressId); + } + + public int getCommits() { + return stats.getInt(commitsId); + } + + public long getCommitTime() { + return stats.getLong(commitTimeId); + } + + public int getCommitsInProgress() { + return stats.getInt(commitsInProgressId); + } + + public Statistics getStats() { + return this.stats; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryFactoryImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryFactoryImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryFactoryImpl.java new file mode 100644 index 0000000..21c019b --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryFactoryImpl.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.lucene.LuceneQuery; +import com.gemstone.gemfire.cache.lucene.LuceneQueryFactory; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; + +public class LuceneQueryFactoryImpl implements LuceneQueryFactory { + private int limit = DEFAULT_LIMIT; + private int pageSize = DEFAULT_PAGESIZE; + private String[] projectionFields = null; + private Cache cache; + + LuceneQueryFactoryImpl(Cache cache) { + this.cache = cache; + } + + @Override + public LuceneQueryFactory setPageSize(int pageSize) { + this.pageSize = pageSize; + return this; + } + + @Override + public LuceneQueryFactory setResultLimit(int limit) { + this.limit = limit; + return this; + } + + @Override + public LuceneQuery create(String indexName, String regionName, String queryString, String defaultField) { + return create(indexName, regionName, new StringQueryProvider(queryString, defaultField)); + } + + @Override + public LuceneQuery create(String indexName, String regionName, LuceneQueryProvider provider) { + Region region = cache.getRegion(regionName); + if(region == null) { + throw new IllegalArgumentException("Region not found: " + regionName); + } + LuceneQueryImpl luceneQuery = new LuceneQueryImpl(indexName, region, provider, projectionFields, limit, pageSize); + return luceneQuery; + } + + @Override + public LuceneQueryFactory setProjectionFields(String... fieldNames) { + projectionFields = fieldNames.clone(); + return this; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryImpl.java new file mode 100644 index 0000000..ef2f4ee --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneQueryImpl.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.execute.Execution; +import com.gemstone.gemfire.cache.execute.FunctionException; +import com.gemstone.gemfire.cache.execute.FunctionService; +import com.gemstone.gemfire.cache.execute.ResultCollector; +import com.gemstone.gemfire.cache.lucene.LuceneQuery; +import com.gemstone.gemfire.cache.lucene.LuceneQueryException; +import com.gemstone.gemfire.cache.lucene.LuceneQueryFactory; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; +import com.gemstone.gemfire.cache.lucene.LuceneResultStruct; +import com.gemstone.gemfire.cache.lucene.PageableLuceneQueryResults; +import com.gemstone.gemfire.cache.lucene.internal.distributed.EntryScore; +import com.gemstone.gemfire.cache.lucene.internal.distributed.LuceneFunction; +import com.gemstone.gemfire.cache.lucene.internal.distributed.LuceneFunctionContext; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntries; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollector; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollectorManager; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesFunctionCollector; + +public class LuceneQueryImpl implements LuceneQuery { + private int limit = LuceneQueryFactory.DEFAULT_LIMIT; + private int pageSize = LuceneQueryFactory.DEFAULT_PAGESIZE; + private String indexName; + // The projected fields are local to a specific index per Query object. + private String[] projectedFieldNames; + /* the lucene Query object to be wrapped here */ + private LuceneQueryProvider query; + private Region region; + private String defaultField; + + public LuceneQueryImpl(String indexName, Region region, LuceneQueryProvider provider, String[] projectionFields, + int limit, int pageSize) { + this.indexName = indexName; + this.region = region; + this.limit = limit; + this.pageSize = pageSize; + this.projectedFieldNames = projectionFields; + this.query = provider; + } + + @Override + public Collection findKeys() throws LuceneQueryException { + TopEntries entries = findTopEntries(); + final List> hits = entries.getHits(); + + return hits.stream() + .map(hit -> hit.getKey()) + .collect(Collectors.toList()); + } + + @Override + public Collection findValues() throws LuceneQueryException { + final List> page = findResults(); + + return page.stream() + .map(entry -> entry.getValue()) + .collect(Collectors.toList()); + } + + @Override + public List> findResults() throws LuceneQueryException { + PageableLuceneQueryResults pages = findPages(0); + if(!pages.hasNext()) { + return Collections.emptyList(); + } + + return pages.next(); + } + + @Override + public PageableLuceneQueryResults findPages() throws LuceneQueryException { + return findPages(pageSize); + } + + private PageableLuceneQueryResults findPages(int pageSize) throws LuceneQueryException { + TopEntries entries = findTopEntries(); + return new PageableLuceneQueryResultsImpl(entries.getHits(), region, pageSize); + } + + private TopEntries findTopEntries() throws LuceneQueryException { + TopEntriesCollectorManager manager = new TopEntriesCollectorManager(null, limit); + LuceneFunctionContext context = new LuceneFunctionContext<>(query, indexName, manager, limit); + TopEntriesFunctionCollector collector = new TopEntriesFunctionCollector(context); + + ResultCollector> rc = (ResultCollector>) onRegion() + .withArgs(context) + .withCollector(collector) + .execute(LuceneFunction.ID); + + //TODO provide a timeout to the user? + TopEntries entries; + try { + entries = rc.getResult(); + } catch(FunctionException e) { + if(e.getCause() instanceof LuceneQueryException) { + throw new LuceneQueryException(e); + } else { + throw e; + } + } + return entries; + } + + protected Execution onRegion() { + return FunctionService.onRegion(region); + } + + @Override + public int getPageSize() { + return this.pageSize; + } + + @Override + public int getLimit() { + return this.limit; + } + + @Override + public String[] getProjectedFieldNames() { + return this.projectedFieldNames; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java new file mode 100755 index 0000000..e708691 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndex.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.lucene.internal.repository.RepositoryManager; +import com.gemstone.gemfire.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; + +public class LuceneRawIndex extends LuceneIndexImpl { + + protected LuceneRawIndex(String indexName, String regionPath, Cache cache) { + super(indexName, regionPath, cache); + } + + @Override + protected RepositoryManager createRepositoryManager() { + HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(getFieldNames()); + return new RawLuceneRepositoryManager(this, mapper); + } + + @Override + public void dumpFiles(String directory) { + return; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java new file mode 100755 index 0000000..6c3bad6 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneRawIndexFactory.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; + +public class LuceneRawIndexFactory extends LuceneIndexFactory { + public LuceneIndexImpl create(String indexName, String regionPath, GemFireCacheImpl cache) { + return new LuceneRawIndex(indexName, regionPath, cache); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneResultStructImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneResultStructImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneResultStructImpl.java new file mode 100644 index 0000000..a3794f2 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneResultStructImpl.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import com.gemstone.gemfire.cache.lucene.LuceneResultStruct; + +public class LuceneResultStructImpl implements LuceneResultStruct { + K key; + V value; + float score; + + public LuceneResultStructImpl(K key, V value, float score) { + this.key = key; + this.value = value; + this.score = score; + } + + @Override + public Object getProjectedField(String fieldName) { + throw new UnsupportedOperationException(); + } + + @Override + public K getKey() { + return key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public float getScore() { + return score; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((key == null) ? 0 : key.hashCode()); + result = prime * result + Float.floatToIntBits(score); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + LuceneResultStructImpl other = (LuceneResultStructImpl) obj; + if (key == null) { + if (other.key != null) + return false; + } else if (!key.equals(other.key)) + return false; + if (Float.floatToIntBits(score) != Float.floatToIntBits(other.score)) + return false; + if (value == null) { + if (other.value != null) + return false; + } else if (!value.equals(other.value)) + return false; + return true; + } + + @Override + public String toString() { + return "LuceneResultStructImpl [key=" + key + ", value=" + value + + ", score=" + score + "]"; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java new file mode 100644 index 0000000..81a62b8 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneServiceImpl.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.util.*; + +import com.gemstone.gemfire.cache.lucene.internal.management.LuceneServiceMBean; +import com.gemstone.gemfire.cache.lucene.internal.management.ManagementIndexListener; +import com.gemstone.gemfire.management.internal.beans.CacheServiceMBeanBase; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +import com.gemstone.gemfire.cache.AttributesFactory; +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.EvictionAlgorithm; +import com.gemstone.gemfire.cache.EvictionAttributes; +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.RegionAttributes; +import com.gemstone.gemfire.cache.execute.FunctionService; +import com.gemstone.gemfire.cache.lucene.LuceneIndex; +import com.gemstone.gemfire.cache.lucene.LuceneQueryFactory; +import com.gemstone.gemfire.cache.lucene.internal.directory.DumpDirectoryFiles; +import com.gemstone.gemfire.cache.lucene.internal.distributed.EntryScore; +import com.gemstone.gemfire.cache.lucene.internal.distributed.LuceneFunction; +import com.gemstone.gemfire.cache.lucene.internal.distributed.LuceneFunctionContext; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntries; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollector; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollectorManager; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.ChunkKey; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.File; +import com.gemstone.gemfire.cache.lucene.internal.xml.LuceneServiceXmlGenerator; +import com.gemstone.gemfire.internal.DSFIDFactory; +import com.gemstone.gemfire.internal.DataSerializableFixedID; +import com.gemstone.gemfire.internal.cache.extension.Extensible; +import com.gemstone.gemfire.internal.cache.CacheService; +import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; +import com.gemstone.gemfire.internal.cache.InternalRegionArguments; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; +import com.gemstone.gemfire.internal.cache.RegionListener; +import com.gemstone.gemfire.internal.cache.xmlcache.XmlGenerator; +import com.gemstone.gemfire.internal.i18n.LocalizedStrings; +import com.gemstone.gemfire.internal.logging.LogService; + +/** + * Implementation of LuceneService to create lucene index and query. + * + * + * @since GemFire 8.5 + */ +public class LuceneServiceImpl implements InternalLuceneService { + public static LuceneIndexFactory luceneIndexFactory = new LuceneIndexFactory(); + private static final Logger logger = LogService.getLogger(); + + private GemFireCacheImpl cache; + private final HashMap indexMap = new HashMap(); + private final HashMap definedIndexMap = new HashMap<>(); + private IndexListener managementListener; + + public LuceneServiceImpl() { + + } + + public void init(final Cache cache) { + if (cache == null) { + throw new IllegalStateException(LocalizedStrings.CqService_CACHE_IS_NULL.toLocalizedString()); + } + GemFireCacheImpl gfc = (GemFireCacheImpl) cache; + gfc.getCancelCriterion().checkCancelInProgress(null); + + this.cache = gfc; + + FunctionService.registerFunction(new LuceneFunction()); + FunctionService.registerFunction(new DumpDirectoryFiles()); + registerDataSerializables(); + } + + @Override + public CacheServiceMBeanBase getMBean() { + LuceneServiceMBean mbean = new LuceneServiceMBean(this); + this.managementListener = new ManagementIndexListener(mbean); + return mbean; + } + + @Override + public Class getInterface() { + return InternalLuceneService.class; + } + + public static String getUniqueIndexName(String indexName, String regionPath) { + if (!regionPath.startsWith("/")) { + regionPath = "/"+regionPath; + } + String name = indexName + "#" + regionPath.replace('/', '_'); + return name; + } + + @Override + public void createIndex(String indexName, String regionPath, String... fields) { + if(fields == null || fields.length == 0) { + throw new IllegalArgumentException("At least one field must be indexed"); + } + StandardAnalyzer analyzer = new StandardAnalyzer(); + + createIndex(indexName, regionPath, analyzer, null, fields); + } + + @Override + public void createIndex(String indexName, String regionPath, Map fieldAnalyzers) { + if(fieldAnalyzers == null || fieldAnalyzers.isEmpty()) { + throw new IllegalArgumentException("At least one field must be indexed"); + } + Analyzer analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), fieldAnalyzers); + Set fieldsSet = fieldAnalyzers.keySet(); + String[] fields = (String[])fieldsSet.toArray(new String[fieldsSet.size()]); + + createIndex(indexName, regionPath, analyzer, fieldAnalyzers, fields); + } + + public void createIndex(final String indexName, String regionPath, + final Analyzer analyzer, final Map fieldAnalyzers, + final String... fields) { + + if(!regionPath.startsWith("/")) { + regionPath = "/" + regionPath; + } + + registerDefinedIndex(LuceneServiceImpl.getUniqueIndexName(indexName, regionPath), + new LuceneIndexCreationProfile(indexName, regionPath, fields, analyzer, fieldAnalyzers)); + + Region region = cache.getRegion(regionPath); + if(region != null) { + definedIndexMap.remove(LuceneServiceImpl.getUniqueIndexName(indexName, regionPath)); + throw new IllegalStateException("The lucene index must be created before region"); + } + + final String dataRegionPath = regionPath; + cache.addRegionListener(new RegionListener() { + @Override + public RegionAttributes beforeCreate(Region parent, String regionName, + RegionAttributes attrs, InternalRegionArguments internalRegionArgs) { + RegionAttributes updatedRA = attrs; + String path = parent == null ? "/" + regionName : parent.getFullPath() + "/" + regionName; + + if(path.equals(dataRegionPath)) { + + if (!attrs.getDataPolicy().withPartitioning()) { + // replicated region + throw new UnsupportedOperationException("Lucene indexes on replicated regions are not supported"); + } + + //For now we cannot support eviction with local destroy. + //Eviction with overflow to disk still needs to be supported + EvictionAttributes evictionAttributes = attrs.getEvictionAttributes(); + EvictionAlgorithm evictionAlgorithm = evictionAttributes.getAlgorithm(); + if (evictionAlgorithm != EvictionAlgorithm.NONE && evictionAttributes.getAction().isLocalDestroy()) { + throw new UnsupportedOperationException("Lucene indexes on regions with eviction and action local destroy are not supported"); + } + + String aeqId = LuceneServiceImpl.getUniqueIndexName(indexName, dataRegionPath); + if (!attrs.getAsyncEventQueueIds().contains(aeqId)) { + AttributesFactory af = new AttributesFactory(attrs); + af.addAsyncEventQueueId(aeqId); + updatedRA = af.create(); + } + + // Add index creation profile + internalRegionArgs.addCacheServiceProfile(new LuceneIndexCreationProfile(indexName, dataRegionPath, fields, analyzer, fieldAnalyzers)); + } + return updatedRA; + } + + @Override + public void afterCreate(Region region) { + if(region.getFullPath().equals(dataRegionPath)) { + afterDataRegionCreated(indexName, analyzer, dataRegionPath, fieldAnalyzers, fields); + cache.removeRegionListener(this); + } + } + }); + } + + + /** + * Finish creating the lucene index after the data region is created . + * + * Public because this is called by the Xml parsing code + */ + public void afterDataRegionCreated(final String indexName, + final Analyzer analyzer, final String dataRegionPath, + final Map fieldAnalyzers, final String... fields) { + LuceneIndexImpl index = createIndexRegions(indexName, dataRegionPath); + index.setSearchableFields(fields); + index.setAnalyzer(analyzer); + index.setFieldAnalyzers(fieldAnalyzers); + index.initialize(); + registerIndex(index); + if (this.managementListener != null) { + this.managementListener.afterIndexCreated(index); + } + } + + private LuceneIndexImpl createIndexRegions(String indexName, String regionPath) { + Region dataregion = this.cache.getRegion(regionPath); + if (dataregion == null) { + logger.info("Data region "+regionPath+" not found"); + return null; + } + //Convert the region name into a canonical form + regionPath = dataregion.getFullPath(); + return luceneIndexFactory.create(indexName, regionPath, cache); + } + + private void registerDefinedIndex(final String regionAndIndex, final LuceneIndexCreationProfile luceneIndexCreationProfile) { + if (definedIndexMap.containsKey(regionAndIndex) || indexMap.containsKey(regionAndIndex)) + throw new IllegalArgumentException("Lucene index already exists in region"); + definedIndexMap.put(regionAndIndex, luceneIndexCreationProfile); + } + + @Override + public LuceneIndex getIndex(String indexName, String regionPath) { + Region region = cache.getRegion(regionPath); + if(region == null) { + return null; + } + return indexMap.get(getUniqueIndexName(indexName, region.getFullPath())); + } + + @Override + public Collection getAllIndexes() { + return indexMap.values(); + } + + @Override + public void destroyIndex(LuceneIndex index) { + LuceneIndexImpl indexImpl = (LuceneIndexImpl) index; + indexMap.remove(getUniqueIndexName(index.getName(), index.getRegionPath())); +// indexImpl.close(); + } + + @Override + public LuceneQueryFactory createLuceneQueryFactory() { + return new LuceneQueryFactoryImpl(cache); + } + + @Override + public XmlGenerator getXmlGenerator() { + return new LuceneServiceXmlGenerator(); + } + + @Override + public void beforeCreate(Extensible source, Cache cache) { + // Nothing to do here. + } + + @Override + public void onCreate(Extensible source, Extensible target) { + //This is called when CacheCreation (source) is turned into a GemfireCacheImpl (target) + //nothing to do there. + } + + public void registerIndex(LuceneIndex index){ + String regionAndIndex = getUniqueIndexName(index.getName(), index.getRegionPath()); + if( !indexMap.containsKey( regionAndIndex )) { + indexMap.put(regionAndIndex, index); + } + definedIndexMap.remove(regionAndIndex); + } + + public void unregisterIndex(final String region){ + if( indexMap.containsKey( region )) indexMap.remove( region ); + } + + /**Public for test purposes */ + public static void registerDataSerializables() { + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_CHUNK_KEY, + ChunkKey.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_FILE, + File.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_FUNCTION_CONTEXT, + LuceneFunctionContext.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_STRING_QUERY_PROVIDER, + StringQueryProvider.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_TOP_ENTRIES_COLLECTOR_MANAGER, + TopEntriesCollectorManager.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_ENTRY_SCORE, + EntryScore.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_TOP_ENTRIES, + TopEntries.class); + + DSFIDFactory.registerDSFID( + DataSerializableFixedID.LUCENE_TOP_ENTRIES_COLLECTOR, + TopEntriesCollector.class); + } + + public Collection getAllDefinedIndexes() { + return definedIndexMap.values(); + } + + public LuceneIndexCreationProfile getDefinedIndex(String indexName, String regionPath) { + return definedIndexMap.get(getUniqueIndexName(indexName , regionPath)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PageableLuceneQueryResultsImpl.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PageableLuceneQueryResultsImpl.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PageableLuceneQueryResultsImpl.java new file mode 100644 index 0000000..1563df0 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PageableLuceneQueryResultsImpl.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.lucene.PageableLuceneQueryResults; +import com.gemstone.gemfire.cache.lucene.LuceneResultStruct; +import com.gemstone.gemfire.cache.lucene.internal.distributed.EntryScore; + +/** + * Implementation of PageableLuceneQueryResults that fetchs a page at a time + * from the server, given a set of EntryScores (key and score). + * + * @param The type of the key + * @param The type of the value + */ +public class PageableLuceneQueryResultsImpl implements PageableLuceneQueryResults { + + /** + * list of docs matching search query + */ + private final List> hits; + + /** + * * Current page of results + */ + private List> currentPage; + /** + * The maximum score. Lazily evaluated + */ + private float maxScore = Float.MIN_VALUE; + + /** + * The user region where values are stored. + */ + private final Region userRegion; + + /** + * The start of the next page of results we want to fetch + */ + private int currentHit = 0; + + /** + * The page size the user wants. + */ + private int pageSize; + + public PageableLuceneQueryResultsImpl(List> hits, Region userRegion, int pageSize) { + this.hits = hits; + this.userRegion = userRegion; + this.pageSize = pageSize == 0 ? Integer.MAX_VALUE : pageSize; + } + + + public List> getHitEntries(int fromIndex, int toIndex) { + List> scores = hits.subList(fromIndex, toIndex); + ArrayList keys = new ArrayList(scores.size()); + for(EntryScore score : scores) { + keys.add(score.getKey()); + } + + Map values = userRegion.getAll(keys); + + ArrayList> results = new ArrayList>(scores.size()); + for(EntryScore score : scores) { + V value = values.get(score.getKey()); + if (value!=null) + results.add(new LuceneResultStructImpl(score.getKey(), value, score.getScore())); + } + return results; + } + + @Override + public List> next() { + if(!hasNext()) { + throw new NoSuchElementException(); + } + List> result = advancePage(); + currentPage = null; + return result; + } + + private List> advancePage() { + if(currentPage != null) { + return currentPage; + } + + int resultSize = (pageSize != Integer.MAX_VALUE) ? pageSize : hits.size(); + currentPage = new ArrayList>(resultSize); + while (currentPage.size() hits.size() ? hits.size() : end; + currentPage.addAll(getHitEntries(currentHit, end)); + currentHit = end; + } + return currentPage; + } + + @Override + public boolean hasNext() { + + advancePage(); + if ( currentPage.isEmpty() ) { + return false; + } + return true; + } + + @Override + public int size() { + return hits.size(); + } + + @Override + public float getMaxScore() { + if(maxScore == Float.MIN_VALUE) { + for(EntryScore score : hits) { + maxScore = Math.max(maxScore, score.getScore()); + } + } + + return maxScore; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java new file mode 100644 index 0000000..d5dd7b1 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/PartitionedRepositoryManager.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.io.IOException; + +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.serializer.LuceneSerializer; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; + +/** + * Manages index repositories for partitioned regions. + * + * This class lazily creates the IndexRepository for each individual + * bucket. If a Bucket is rebalanced, this class will create a new + * index repository when the bucket returns to this node. + */ +public class PartitionedRepositoryManager extends AbstractPartitionedRepositoryManager { + + public static IndexRepositoryFactory indexRepositoryFactory = new IndexRepositoryFactory(); + + public PartitionedRepositoryManager(LuceneIndexImpl index, + LuceneSerializer serializer) { + super(index, serializer); + } + + @Override + public IndexRepository createOneIndexRepository(Integer bucketId, + LuceneSerializer serializer, LuceneIndexImpl index, + PartitionedRegion userRegion) throws IOException { + return indexRepositoryFactory.createIndexRepository(bucketId, serializer, index, userRegion); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java new file mode 100755 index 0000000..131e297 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawIndexRepositoryFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import java.io.File; +import java.io.IOException; + +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.store.RAMDirectory; + +import com.gemstone.gemfire.cache.lucene.internal.directory.RegionDirectory; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepositoryImpl; +import com.gemstone.gemfire.cache.lucene.internal.repository.serializer.LuceneSerializer; +import com.gemstone.gemfire.internal.cache.BucketRegion; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; + +public class RawIndexRepositoryFactory extends IndexRepositoryFactory { + public RawIndexRepositoryFactory() { + } + + public IndexRepository createIndexRepository(final Integer bucketId, + LuceneSerializer serializer, + LuceneIndexImpl index, PartitionedRegion userRegion) + throws IOException + { + final IndexRepository repo; + LuceneRawIndex indexForRaw = (LuceneRawIndex)index; + BucketRegion dataBucket = getMatchingBucket(userRegion, bucketId); + + Directory dir = null; + if (indexForRaw.withPersistence()) { + String bucketLocation = LuceneServiceImpl.getUniqueIndexName(index.getName(), index.getRegionPath()+"_"+bucketId); + File location = new File(index.getName(), bucketLocation); + if (!location.exists()) { + location.mkdirs(); + } + dir = new NIOFSDirectory(location.toPath()); + } else { + dir = new RAMDirectory(); + } + IndexWriterConfig config = new IndexWriterConfig(indexForRaw.getAnalyzer()); + IndexWriter writer = new IndexWriter(dir, config); + return new IndexRepositoryImpl(null, writer, serializer, indexForRaw.getIndexStats(), dataBucket); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java new file mode 100755 index 0000000..234245e --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/RawLuceneRepositoryManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal; + +import java.io.IOException; + +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.serializer.LuceneSerializer; +import com.gemstone.gemfire.internal.cache.BucketNotFoundException; +import com.gemstone.gemfire.internal.cache.PartitionedRegion; + +public class RawLuceneRepositoryManager extends AbstractPartitionedRepositoryManager { + public static IndexRepositoryFactory indexRepositoryFactory = new RawIndexRepositoryFactory(); + + public RawLuceneRepositoryManager(LuceneIndexImpl index, + LuceneSerializer serializer) { + super(index, serializer); + } + + public void close() { + for (IndexRepository repo:indexRepositories.values()) { + repo.cleanup(); + } + } + + @Override + public IndexRepository createOneIndexRepository(Integer bucketId, + LuceneSerializer serializer, LuceneIndexImpl index, + PartitionedRegion userRegion) throws IOException { + return indexRepositoryFactory.createIndexRepository(bucketId, serializer, index, userRegion); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/StringQueryProvider.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/StringQueryProvider.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/StringQueryProvider.java new file mode 100644 index 0000000..c5d145e --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/StringQueryProvider.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; +import org.apache.lucene.search.Query; + +import com.gemstone.gemfire.DataSerializer; +import com.gemstone.gemfire.GemFireCheckedException; +import com.gemstone.gemfire.GemFireException; +import com.gemstone.gemfire.cache.lucene.LuceneIndex; +import com.gemstone.gemfire.cache.lucene.LuceneQueryException; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; +import com.gemstone.gemfire.cache.query.QueryException; +import com.gemstone.gemfire.internal.DataSerializableFixedID; +import com.gemstone.gemfire.internal.Version; +import com.gemstone.gemfire.internal.logging.LogService; + +/** + * Constructs a Lucene Query object by parsing a search string. The class uses {@link StandardQueryParser}. It sets + * searchable fields in a {@link LuceneIndex} as default fields. + */ +public class StringQueryProvider implements LuceneQueryProvider, DataSerializableFixedID { + + private static final long serialVersionUID = 1L; + + private static final Logger logger = LogService.getLogger(); + + // the following members hold input data and needs to be sent on wire + private String query; + + // the following members hold derived objects and need not be serialized + private transient Query luceneQuery; + + private String defaultField; + + public StringQueryProvider() { + this(null, null); + } + + public StringQueryProvider(String query, String defaultField) { + this.query = query; + this.defaultField = defaultField; + } + + @Override + public synchronized Query getQuery(LuceneIndex index) throws LuceneQueryException { + if (luceneQuery == null) { + String[] fields = index.getFieldNames(); + LuceneIndexImpl indexImpl = (LuceneIndexImpl) index; + StandardQueryParser parser = new StandardQueryParser(indexImpl.getAnalyzer()); + try { + luceneQuery = parser.parse(query, defaultField); + if (logger.isDebugEnabled()) { + logger.debug("User query " + query + " is parsed to be: " + luceneQuery); + } + } catch (QueryNodeException e) { + logger.debug("Query node exception:" + query, e); + throw new LuceneQueryException("Malformed lucene query: " + query, e); + } + } + return luceneQuery; + } + + /** + * @return the query string used to construct this query provider + */ + public String getQueryString() { + return query; + } + + @Override + public Version[] getSerializationVersions() { + return null; + } + + @Override + public int getDSFID() { + return LUCENE_STRING_QUERY_PROVIDER; + } + + @Override + public void toData(DataOutput out) throws IOException { + DataSerializer.writeString(query, out); + DataSerializer.writeString(defaultField, out); + } + + @Override + public void fromData(DataInput in) throws IOException, ClassNotFoundException { + query = DataSerializer.readString(in); + defaultField = DataSerializer.readString(in); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneCliStrings.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneCliStrings.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneCliStrings.java new file mode 100644 index 0000000..b424d8f --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneCliStrings.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal.cli; + +public class LuceneCliStrings { + //Common parameters/options + public static final String LUCENE__INDEX_NAME = "name"; + public static final String LUCENE__REGION_PATH = "region"; + + //List lucene index commands + public static final String LUCENE_LIST_INDEX = "list lucene indexes"; + public static final String LUCENE_LIST_INDEX__HELP = "Display the list of lucene indexes created for all members."; + public static final String LUCENE_LIST_INDEX__ERROR_MESSAGE = "An error occurred while collecting all lucene index information across the Geode cluster: %1$s"; + public static final String LUCENE_LIST_INDEX__INDEXES_NOT_FOUND_MESSAGE = "No lucene indexes found"; + public static final String LUCENE_LIST_INDEX__STATS = "with-stats"; + public static final String LUCENE_LIST_INDEX__STATS__HELP = "Display lucene index stats"; + + //Create lucene index commands + public static final String LUCENE_CREATE_INDEX = "create lucene index"; + public static final String LUCENE_CREATE_INDEX__HELP = "Create a lucene index that can be used to execute queries."; + public static final String LUCENE_CREATE_INDEX__NAME__HELP = "Name of the lucene index to create."; + public static final String LUCENE_CREATE_INDEX__REGION_HELP = "Name/Path of the region where the lucene index is created on."; + public static final String LUCENE_CREATE_INDEX__FIELD = "field"; + public static final String LUCENE_CREATE_INDEX__FIELD_HELP = "fields on the region values which are stored in the lucene index."; + public static final String LUCENE_CREATE_INDEX__ANALYZER = "analyzer"; + public static final String LUCENE_CREATE_INDEX__ANALYZER_HELP = "Type of the analyzer for each field."; + public static final String LUCENE_CREATE_INDEX__GROUP = "group"; + public static final String LUCENE_CREATE_INDEX__GROUP__HELP = "Group of members in which the lucene index will be created."; + public static final String CREATE_INDEX__SUCCESS__MSG = "Index successfully created with following details"; + public static final String CREATE_INDEX__FAILURE__MSG = "Failed to create index \"{0}\" due to following reasons"; + public static final String CREATE_INDEX__NAME__MSG = "Name : {0}"; + public static final String CREATE_INDEX__REGIONPATH__MSG = "RegionPath : {0}"; + public static final String CREATE_INDEX__MEMBER__MSG = "Members which contain the index"; + public static final String CREATE_INDEX__NUMBER__AND__MEMBER = "{0}. {1}"; + public static final String CREATE_INDEX__EXCEPTION__OCCURRED__ON = "Occurred on following members"; + + //Describe lucene index commands + public static final String LUCENE_DESCRIBE_INDEX = "describe lucene index"; + public static final String LUCENE_DESCRIBE_INDEX__HELP = "Display the describe of lucene indexes created for all members."; + public static final String LUCENE_DESCRIBE_INDEX__ERROR_MESSAGE = "An error occurred while collecting lucene index information across the Geode cluster: %1$s"; + public static final String LUCENE_DESCRIBE_INDEX__NAME__HELP = "Name of the lucene index to describe."; + public static final String LUCENE_DESCRIBE_INDEX__REGION_HELP = "Name/Path of the region where the lucene index to be described exists."; + + + //Search lucene index commands + public static final String LUCENE_SEARCH_INDEX = "search lucene"; + public static final String LUCENE_SEARCH_INDEX__HELP = "Search lucene index"; + public static final String LUCENE_SEARCH_INDEX__ERROR_MESSAGE = "An error occurred while searching lucene index across the Geode cluster: %1$s"; + public static final String LUCENE_SEARCH_INDEX__NAME__HELP = "Name of the lucene index to search."; + public static final String LUCENE_SEARCH_INDEX__REGION_HELP = "Name/Path of the region where the lucene index exists."; + public static final String LUCENE_SEARCH_INDEX__QUERY_STRING="queryStrings"; + public static final String LUCENE_SEARCH_INDEX__LIMIT="limit"; + public static final String LUCENE_SEARCH_INDEX__LIMIT__HELP="Number of search results needed"; + public static final String LUCENE_SEARCH_INDEX__QUERY_STRING__HELP="Query string to search the lucene index"; + public static final String LUCENE_SEARCH_INDEX__DEFAULT_FIELD="defaultField"; + public static final String LUCENE_SEARCH_INDEX__DEFAULT_FIELD__HELP="Default field to search in"; + public static final String LUCENE_SEARCH_INDEX__NO_RESULTS_MESSAGE="No results"; + public static final String LUCENE_SEARCH_INDEX__PAGE_SIZE="pageSize"; + public static final String LUCENE_SEARCH_INDEX__PAGE_SIZE__HELP="Number of results to be returned in a page"; + public static final String LUCENE_SEARCH_INDEX__KEYSONLY="keys-only"; + public static final String LUCENE_SEARCH_INDEX__KEYSONLY__HELP="Return only keys of search results."; + +}