geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject [11/17] incubator-geode git commit: GEODE-11: Support indexing values that are Strings or Numbers
Date Thu, 16 Jun 2016 22:07:25 GMT
GEODE-11: Support indexing values that are Strings or Numbers

Adding support to index values that are strings or numbers, by providing
a special field name LuceneIndex.REGION_VALUE_FIELD that indicates the
entire value should be indexed.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/fae0f90f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/fae0f90f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/fae0f90f

Branch: refs/heads/feature/GEODE-420
Commit: fae0f90fa1652d30b1e1fe90eb2e1baa2f2e835c
Parents: aef3353
Author: Dan Smith <upthewaterspout@apache.org>
Authored: Mon Jun 13 16:49:23 2016 -0700
Committer: Udo Kohlmeyer <ukohlmeyer@pivotal.io>
Committed: Fri Jun 17 08:05:58 2016 +1000

----------------------------------------------------------------------
 .../gemfire/cache/lucene/LuceneService.java     | 16 +++++-
 .../lucene/internal/LuceneServiceImpl.java      |  6 +++
 .../HeterogeneousLuceneSerializer.java          | 22 +++++++-
 .../serializer/PrimitiveSerializer.java         | 35 +++++++++++++
 .../repository/serializer/SerializerUtil.java   | 30 +++++++++--
 .../lucene/LuceneQueriesIntegrationTest.java    | 13 +++++
 .../internal/LuceneServiceImplJUnitTest.java    | 53 ++++++++++++++++++++
 .../HeterogeneousLuceneSerializerJUnitTest.java | 21 ++++++++
 8 files changed, 188 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
index 462aa7e..974cf96 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/LuceneService.java
@@ -68,9 +68,20 @@ import com.gemstone.gemfire.cache.GemFireCache;
  */
 @Experimental
 public interface LuceneService {
-  
+
+  /**
+   * A special field name that indicates that the entire region value should
+   * be indexed. This will only work if the region value is a String or Number, in
+   * which case a lucene document will be created with a single field with this name.
+   */
+  String REGION_VALUE_FIELD = "__REGION_VALUE_FIELD";
+
   /**
    * Create a lucene index using default analyzer.
+   * @param fields The fields of the object to index. Only fields listed here will be stored
+   * in the index. Fields should map to PDX fieldNames if the object is serialized with PDX,
or
+   * to java fields on the object otherwise. The special field name {{@link #REGION_VALUE_FIELD}}
+   * indicates that the entire value should be stored as a single field in the index.
    */
   public void createIndex(String indexName, String regionPath, String... fields);
   
@@ -79,7 +90,8 @@ public interface LuceneService {
    * 
    * @param indexName index name
    * @param regionPath region name
-   * @param analyzerPerField analyzer per field map
+   * @param analyzerPerField A map of fields to analyzers. See {{@link #createIndex(String,
String, String...)}}
+   * for details on valid values for fields. Each field will be tokenized using the provided
Analyzer.
    */
   public void createIndex(String indexName, String regionPath,
       Map<String, Analyzer> analyzerPerField);

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
index 47c4d76..6bfaa78 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImpl.java
@@ -102,6 +102,9 @@ public class LuceneServiceImpl implements InternalLuceneService {
 
   @Override
   public void createIndex(String indexName, String regionPath, String... fields) {
+    if(fields == null || fields.length == 0) {
+      throw new IllegalArgumentException("At least one field must be indexed");
+    }
     StandardAnalyzer analyzer = new StandardAnalyzer();
     
     createIndex(indexName, regionPath, analyzer, null, fields);
@@ -109,6 +112,9 @@ public class LuceneServiceImpl implements InternalLuceneService {
   
   @Override
   public void createIndex(String indexName, String regionPath, Map<String, Analyzer>
fieldAnalyzers) {
+    if(fieldAnalyzers == null || fieldAnalyzers.isEmpty()) {
+      throw new IllegalArgumentException("At least one field must be indexed");
+    }
     Analyzer analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(), fieldAnalyzers);
     Set<String> fieldsSet = fieldAnalyzers.keySet();
     String[] fields = (String[])fieldsSet.toArray(new String[fieldsSet.size()]);

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
index d2b1db1..271f0fd 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializer.java
@@ -18,11 +18,14 @@
  */
 package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
 
+import java.util.Arrays;
 import java.util.Map;
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.document.Document;
 
+import com.gemstone.gemfire.cache.lucene.LuceneIndex;
+import com.gemstone.gemfire.cache.lucene.LuceneService;
 import com.gemstone.gemfire.internal.logging.LogService;
 import com.gemstone.gemfire.internal.util.concurrent.CopyOnWriteWeakHashMap;
 import com.gemstone.gemfire.pdx.PdxInstance;
@@ -42,7 +45,7 @@ public class HeterogeneousLuceneSerializer implements LuceneSerializer {
    * A mapper for converting a PDX object into a document
    */
   private LuceneSerializer pdxMapper;
-  
+
   /**
    * Mappers for each individual class type that this class has seen.
    * 
@@ -55,8 +58,23 @@ public class HeterogeneousLuceneSerializer implements LuceneSerializer
{
   public HeterogeneousLuceneSerializer(String[] indexedFields) {
     this.indexedFields = indexedFields;
     pdxMapper = new PdxLuceneSerializer(indexedFields);
+
+
+    addSerializersForPrimitiveValues();
   }
-  
+
+  /**
+   * Add serializers for the primitive value types (String, Number, etc.)
+   * if the user has requested that the whole value be serialized
+   */
+  private void addSerializersForPrimitiveValues() {
+    if(Arrays.asList(indexedFields).contains(LuceneService.REGION_VALUE_FIELD)) {
+      final PrimitiveSerializer primitiveSerializer = new PrimitiveSerializer();
+      SerializerUtil.supportedPrimitiveTypes().stream()
+        .forEach(type -> mappers.put(type, primitiveSerializer));
+    }
+  }
+
   @Override
   public void toDocument(Object value, Document doc) {
     

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
new file mode 100644
index 0000000..d096637
--- /dev/null
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/PrimitiveSerializer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
+
+import com.gemstone.gemfire.cache.lucene.LuceneService;
+
+import org.apache.lucene.document.Document;
+
+/**
+ * A LuceneSerializer that can serialize a primitive value (String, int, long, double)
+ * by creating a document with a special field containing the value
+ */
+public class PrimitiveSerializer implements LuceneSerializer {
+
+  @Override
+  public void toDocument(final Object value, final Document doc) {
+    SerializerUtil.addField(doc, LuceneService.REGION_VALUE_FIELD, value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
index 0ed9d5d..4d563c1 100644
--- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
+++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/SerializerUtil.java
@@ -22,6 +22,10 @@ package com.gemstone.gemfire.cache.lucene.internal.repository.serializer;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DoublePoint;
@@ -44,6 +48,23 @@ import com.gemstone.gemfire.internal.util.BlobHelper;
  */
 public class SerializerUtil {
   private static final String KEY_FIELD = "_KEY";
+
+  private static final Set<Class> SUPPORTED_PRIMITIVE_TYPES;
+
+  static {
+    HashSet<Class> primitiveTypes = new HashSet<>();
+    primitiveTypes.add(String.class);
+    primitiveTypes.add(long.class);
+    primitiveTypes.add(int.class);
+    primitiveTypes.add(float.class);
+    primitiveTypes.add(double.class);
+    primitiveTypes.add(Long.class);
+    primitiveTypes.add(Integer.class);
+    primitiveTypes.add(Float.class);
+    primitiveTypes.add(Double.class);
+
+    SUPPORTED_PRIMITIVE_TYPES = Collections.unmodifiableSet(primitiveTypes);
+  }
   
   /**
    * A small buffer for converting keys to byte[] arrays.
@@ -97,10 +118,11 @@ public class SerializerUtil {
    * Return true if a field type can be written to a lucene document.
    */
   public static boolean isSupported(Class<?> type) {
-    return type == String.class || type == long.class || type == int.class 
-        || type == float.class || type == double.class
-        || type == Long.class || type == Integer.class 
-        || type == Float.class || type == Double.class; 
+    return SUPPORTED_PRIMITIVE_TYPES.contains(type);
+  }
+
+  public static Collection<Class> supportedPrimitiveTypes() {
+    return SUPPORTED_PRIMITIVE_TYPES;
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
index 587e4db..674bc9c 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -186,6 +186,19 @@ public class LuceneQueriesIntegrationTest extends LuceneIntegrationTest
{
   }
 
   @Test()
+  public void shouldAllowQueryOnRegionWithStringValue() throws ParseException {
+    luceneService.createIndex(INDEX_NAME, REGION_NAME, LuceneService.REGION_VALUE_FIELD);
+    Region region = cache.createRegionFactory(RegionShortcut.PARTITION)
+      .create(REGION_NAME);
+    final LuceneIndex index = luceneService.getIndex(INDEX_NAME, REGION_NAME);
+
+    region.put("A", "one three");
+    index.waitUntilFlushed(60000);
+
+    verifyQuery("one", LuceneService.REGION_VALUE_FIELD, "A");
+  }
+
+  @Test()
   public void throwFunctionExceptionWhenGivenBadQuery() {
     LuceneService luceneService = LuceneServiceProvider.get(cache);
     luceneService.createIndex(INDEX_NAME, REGION_NAME, "text");

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
new file mode 100644
index 0000000..3f96323
--- /dev/null
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneServiceImplJUnitTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gemstone.gemfire.cache.lucene.internal;
+
+import static org.junit.Assert.*;
+
+import java.util.Collections;
+
+import com.gemstone.gemfire.cache.lucene.LuceneService;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+@Category(UnitTest.class)
+public class LuceneServiceImplJUnitTest {
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  public void shouldThrowIllegalArgumentExceptionIfFieldsAreMissing() {
+    LuceneServiceImpl service = new LuceneServiceImpl();
+    thrown.expect(IllegalArgumentException.class);
+    service.createIndex("index", "region");
+  }
+
+  @Test
+  public void shouldThrowIllegalArgumentExceptionIfFieldsMapIsMissing() {
+    LuceneServiceImpl service = new LuceneServiceImpl();
+    thrown.expect(IllegalArgumentException.class);
+    service.createIndex("index", "region", Collections.emptyMap());
+  }
+
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/fae0f90f/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
index 3bce904..7b32ab2 100644
--- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
+++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/repository/serializer/HeterogeneousLuceneSerializerJUnitTest.java
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Document;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.gemstone.gemfire.cache.lucene.LuceneService;
 import com.gemstone.gemfire.pdx.PdxInstance;
 import com.gemstone.gemfire.test.junit.categories.UnitTest;
 
@@ -83,4 +84,24 @@ public class HeterogeneousLuceneSerializerJUnitTest {
     assertEquals(5, doc3.getField("i").numericValue());
   }
 
+  @Test
+  public void shouldIndexPrimitiveStringIfRequested() {
+    HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(new String[]
{LuceneService.REGION_VALUE_FIELD});
+    Document doc = new Document();
+    mapper.toDocument("sample value", doc);
+    assertEquals(1, doc.getFields().size());
+    assertEquals("sample value", doc.getField(LuceneService.REGION_VALUE_FIELD).stringValue());
+  }
+
+  @Test
+  public void shouldIndexPrimitiveNumberIfRequested() {
+    HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(new String[]
{LuceneService.REGION_VALUE_FIELD});
+    Document doc = new Document();
+    mapper.toDocument(53, doc);
+
+    assertEquals(1, doc.getFields().size());
+
+    assertEquals(53, doc.getField(LuceneService.REGION_VALUE_FIELD).numericValue());
+  }
+
 }


Mime
View raw message