accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ujustgotbi...@apache.org
Subject [01/50] git commit: ACCUMULO-354 added boolean instead of null to detect presence of next value
Date Thu, 06 Feb 2014 05:39:40 GMT
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT [created] e15859054


ACCUMULO-354 added boolean instead of null to detect presence of next value

git-svn-id: https://svn.apache.org/repos/asf/incubator/accumulo/branches/1.4@1238696 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/commit/b9cf2945
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/tree/b9cf2945
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/diff/b9cf2945

Branch: refs/heads/1.4.5-SNAPSHOT
Commit: b9cf2945ee33ace0726298601462d02b0e226190
Parents: 72fbb54
Author: Billie Rinaldi <billie@apache.org>
Authored: Tue Jan 31 16:53:40 2012 +0000
Committer: Billie Rinaldi <billie@apache.org>
Committed: Tue Jan 31 16:53:40 2012 +0000

----------------------------------------------------------------------
 .../wikisearch/ingest/WikipediaIngester.java    |  3 ++
 .../wikisearch/iterator/TextIndexTest.java      | 43 ++++++++++++++++++++
 2 files changed, 46 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/b9cf2945/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
----------------------------------------------------------------------
diff --git a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
index 31c8472..50415a7 100644
--- a/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
+++ b/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
@@ -88,6 +88,7 @@ public class WikipediaIngester extends Configured implements Tool {
           columns.add(new Column("fi\0" + family));
         }
         TextIndexCombiner.setColumns(setting, columns);
+        TextIndexCombiner.setLossyness(setting, true);
         
         tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
       }
@@ -102,6 +103,7 @@ public class WikipediaIngester extends Configured implements Tool {
       // Add the UID combiner
       IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
       GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+      GlobalIndexUidCombiner.setLossyness(setting, true);
       tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
     }
     
@@ -110,6 +112,7 @@ public class WikipediaIngester extends Configured implements Tool {
       // Add the UID combiner
       IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
       GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+      GlobalIndexUidCombiner.setLossyness(setting, true);
       tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
     }
     

http://git-wip-us.apache.org/repos/asf/accumulo-wikisearch/blob/b9cf2945/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
----------------------------------------------------------------------
diff --git a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
index 22ef9aa..7297b5a 100644
--- a/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
+++ b/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
@@ -139,4 +139,47 @@ public class TextIndexTest {
     Assert.assertTrue(offsets.get(4) == 15);
     Assert.assertTrue(offsets.get(5) == 19);
   }
+  
+  @Test
+  public void testEmptyValue() throws InvalidProtocolBufferException {
+    Builder builder = createBuilder();
+    builder.addWordOffset(13);
+    builder.addWordOffset(15);
+    builder.addWordOffset(19);
+    builder.setNormalizedTermFrequency(0.12f);
+    
+    values.add(new Value("".getBytes()));
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    builder = createBuilder();
+    builder.addWordOffset(1);
+    builder.addWordOffset(5);
+    builder.setNormalizedTermFrequency(0.1f);
+    
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    builder = createBuilder();
+    builder.addWordOffset(3);
+    builder.setNormalizedTermFrequency(0.05f);
+    
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    Value result = combiner.reduce(new Key(), values.iterator());
+    
+    TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
+    
+    Assert.assertTrue(info.getNormalizedTermFrequency() == 0.27f);
+    
+    List<Integer> offsets = info.getWordOffsetList();
+    Assert.assertTrue(offsets.size() == 6);
+    Assert.assertTrue(offsets.get(0) == 1);
+    Assert.assertTrue(offsets.get(1) == 3);
+    Assert.assertTrue(offsets.get(2) == 5);
+    Assert.assertTrue(offsets.get(3) == 13);
+    Assert.assertTrue(offsets.get(4) == 15);
+    Assert.assertTrue(offsets.get(5) == 19);
+  }
 }


Mime
View raw message