accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1238754 - in /incubator/accumulo/trunk: ./ src/core/ src/core/src/main/java/org/apache/accumulo/core/iterators/ src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/ src/examples/wikisearch/ingest/src...
Date Tue, 31 Jan 2012 19:21:03 GMT
Author: ecn
Date: Tue Jan 31 19:21:02 2012
New Revision: 1238754

URL: http://svn.apache.org/viewvc?rev=1238754&view=rev
Log:
ACCUMULO-357, ACCUMULO-354, ACCUMULO-356, merge to trunk

Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/   (props changed)
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
    incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
    incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
    incubator/accumulo/trunk/src/server/   (props changed)
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
    incubator/accumulo/trunk/test/system/auto/TestUtils.py

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1237930
+/incubator/accumulo/branches/1.4:1201902-1238753

Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/core:1209938
 /incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1237930
+/incubator/accumulo/branches/1.3.5rc/src/core:1209938
+/incubator/accumulo/branches/1.4/src/core:1201902-1238753

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
Tue Jan 31 19:21:02 2012
@@ -40,7 +40,7 @@ public abstract class TypedValueCombiner
   private boolean lossy = false;
   
   protected static final String LOSSY = "lossy";
-
+  
   /**
    * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using
the decode method of an Encoder.
    */
@@ -68,19 +68,19 @@ public abstract class TypedValueCombiner
     }
     
     V next = null;
+    boolean hasNext = false;
+    
     @Override
     public boolean hasNext() {
-      if (next != null)
+      if (hasNext)
         return true;
-
-      while (true)
-      {
+      
+      while (true) {
         if (!source.hasNext())
           return false;
-        try
-        {
+        try {
           next = encoder.decode(source.next().get());
-          return true;
+          return hasNext = true;
         } catch (ValueFormatException vfe) {
           if (!lossy)
             throw vfe;
@@ -90,10 +90,11 @@ public abstract class TypedValueCombiner
     
     @Override
     public V next() {
-      if (!hasNext())
+      if (!hasNext && !hasNext())
         throw new NoSuchElementException();
       V toRet = next;
       next = null;
+      hasNext = false;
       return toRet;
     }
     
@@ -192,7 +193,7 @@ public abstract class TypedValueCombiner
     super.init(source, options, env);
     setLossyness(options);
   }
-
+  
   private void setLossyness(Map<String,String> options) {
     String loss = options.get(LOSSY);
     if (loss == null)
@@ -214,7 +215,7 @@ public abstract class TypedValueCombiner
     setLossyness(options);
     return true;
   }
-
+  
   /**
    * A convenience method to set the "lossy" option on a TypedValueCombiner. If true, the
combiner will ignore any values which fail to decode. Otherwise, the
    * combiner will throw an error which will interrupt the action (and prevent potential
data loss). False is the default behavior.

Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
(original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/main/java/org/apache/accumulo/examples/wikisearch/ingest/WikipediaIngester.java
Tue Jan 31 19:21:02 2012
@@ -88,6 +88,7 @@ public class WikipediaIngester extends C
           columns.add(new Column("fi\0" + family));
         }
         TextIndexCombiner.setColumns(setting, columns);
+        TextIndexCombiner.setLossyness(setting, true);
         
         tops.attachIterator(tableName, setting, EnumSet.allOf(IteratorScope.class));
       }
@@ -102,6 +103,7 @@ public class WikipediaIngester extends C
       // Add the UID combiner
       IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
       GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+      GlobalIndexUidCombiner.setLossyness(setting, true);
       tops.attachIterator(indexTableName, setting, EnumSet.allOf(IteratorScope.class));
     }
     
@@ -110,6 +112,7 @@ public class WikipediaIngester extends C
       // Add the UID combiner
       IteratorSetting setting = new IteratorSetting(19, "UIDAggregator", GlobalIndexUidCombiner.class);
       GlobalIndexUidCombiner.setCombineAllColumns(setting, true);
+      GlobalIndexUidCombiner.setLossyness(setting, true);
       tops.attachIterator(reverseIndexTableName, setting, EnumSet.allOf(IteratorScope.class));
     }
     

Modified: incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
(original)
+++ incubator/accumulo/trunk/src/examples/wikisearch/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/iterator/TextIndexTest.java
Tue Jan 31 19:21:02 2012
@@ -139,4 +139,47 @@ public class TextIndexTest {
     Assert.assertTrue(offsets.get(4) == 15);
     Assert.assertTrue(offsets.get(5) == 19);
   }
+  
+  @Test
+  public void testEmptyValue() throws InvalidProtocolBufferException {
+    Builder builder = createBuilder();
+    builder.addWordOffset(13);
+    builder.addWordOffset(15);
+    builder.addWordOffset(19);
+    builder.setNormalizedTermFrequency(0.12f);
+    
+    values.add(new Value("".getBytes()));
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    builder = createBuilder();
+    builder.addWordOffset(1);
+    builder.addWordOffset(5);
+    builder.setNormalizedTermFrequency(0.1f);
+    
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    builder = createBuilder();
+    builder.addWordOffset(3);
+    builder.setNormalizedTermFrequency(0.05f);
+    
+    values.add(new Value(builder.build().toByteArray()));
+    values.add(new Value("".getBytes()));
+    
+    Value result = combiner.reduce(new Key(), values.iterator());
+    
+    TermWeight.Info info = TermWeight.Info.parseFrom(result.get());
+    
+    Assert.assertTrue(info.getNormalizedTermFrequency() == 0.27f);
+    
+    List<Integer> offsets = info.getWordOffsetList();
+    Assert.assertTrue(offsets.size() == 6);
+    Assert.assertTrue(offsets.get(0) == 1);
+    Assert.assertTrue(offsets.get(1) == 3);
+    Assert.assertTrue(offsets.get(2) == 5);
+    Assert.assertTrue(offsets.get(3) == 13);
+    Assert.assertTrue(offsets.get(4) == 15);
+    Assert.assertTrue(offsets.get(5) == 19);
+  }
 }

Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 31 19:21:02 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3.5rc/src/server:1209938
 /incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1237930
+/incubator/accumulo/branches/1.3.5rc/src/server:1209938
+/incubator/accumulo/branches/1.4/src/server:1201902-1238753

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Tue Jan 31 19:21:02 2012
@@ -1490,22 +1490,23 @@ public class Master implements LiveTServ
       for (MergeStats stats : mergeStatsCache.values()) {
         try {
           MergeState update = stats.nextMergeState();
-          if (update != stats.getMergeInfo().getState()) {
-            if (update == MergeState.MERGING) {
-              if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
-                try {
-                  if (stats.getMergeInfo().isDelete())
-                    deleteTablets(stats.getMergeInfo());
-                  else
-                    mergeMetadataRecords(stats.getMergeInfo());
-                  setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
-                } catch (Exception ex) {
-                  log.error("Unable merge metadata table records", ex);
+          if (update == MergeState.MERGING) {
+            if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
+              try {
+                if (stats.getMergeInfo().isDelete()) {
+                  deleteTablets(stats.getMergeInfo());
+                } else {
+                  mergeMetadataRecords(stats.getMergeInfo());
                 }
+                setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
+              } catch (Exception ex) {
+                log.error("Unable merge metadata table records", ex);
               }
             }
-            if (update == MergeState.COMPLETE)
-              update = MergeState.NONE;
+          }
+          if (update == MergeState.COMPLETE)
+            update = MergeState.NONE;
+          if (update != stats.getMergeInfo().getState()) {
             setMergeState(stats.getMergeInfo(), update);
           }
         } catch (Exception ex) {

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
Tue Jan 31 19:21:02 2012
@@ -90,9 +90,14 @@ public class MergeStats {
   
   public MergeState nextMergeState() throws Exception {
     MergeState state = info.getState();
+    log.info("Computing next merge state for " + this.info.getRange() + " which is presently
" + state);
     if (state == MergeState.STARTED) {
       state = MergeState.SPLITTING;
     }
+    if (total == 0) {
+      log.info("failed to see any tablets for this range, ignoring");
+      return state;
+    }
     if (state == MergeState.SPLITTING) {
       log.info(hosted + " are hosted, total " + total);
       if (!info.isDelete() && total == 1) {
@@ -171,9 +176,11 @@ public class MergeStats {
         break;
       }
       verify.update(tls.extent, tls.getState(master.onlineTabletServers()), tls.chopped);
+      // check that the prevRow matches the previous row
       if (pr != null && (tls.extent.getPrevEndRow() == null || !tls.extent.getPrevEndRow().equals(pr)))
         return false;
       pr = tls.extent.getEndRow();
+      // stop when we've seen the tablet just beyond our range
       if (tls.extent.getPrevEndRow() != null && extent.getEndRow() != null &&
tls.extent.getPrevEndRow().compareTo(extent.getEndRow()) > 0) {
         break;
       }
@@ -184,7 +191,8 @@ public class MergeStats {
   public static void main(String[] args) throws Exception {
     Instance instance = HdfsZooInstance.getInstance();
     Map<String,String> tableIdMap = instance.getConnector(SecurityConstants.getSystemCredentials()).tableOperations().tableIdMap();
-    for (String tableId : tableIdMap.keySet()) {
+    for (String table : tableIdMap.keySet()) {
+      String tableId = tableIdMap.get(table);
       String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" +
tableId.toString() + "/merge";
       MergeInfo info = new MergeInfo();
       if (ZooReaderWriter.getInstance().exists(path)) {
@@ -193,7 +201,7 @@ public class MergeStats {
         in.reset(data, data.length);
         info.readFields(in);
       }
-      System.out.println(String.format("%25s  %10s %10s %s", tableIdMap.get(tableId), info.state,
info.operation, info.range));
+      System.out.println(String.format("%25s  %10s %10s %s", table, info.state, info.operation,
info.range));
     }
   }
 }
\ No newline at end of file

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/security/ZKAuthenticator.java
Tue Jan 31 19:21:02 2012
@@ -370,7 +370,10 @@ public final class ZKAuthenticator imple
       if (Tool.convertSystemPermissions(perms).contains(permission))
         return true;
       zooCache.clear(ZKUserPath + "/" + user + ZKUserSysPerms);
-      return Tool.convertSystemPermissions(zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms)).contains(permission);
+      perms = zooCache.get(ZKUserPath + "/" + user + ZKUserSysPerms);
+      if (perms == null)
+        return false;
+      return Tool.convertSystemPermissions(perms).contains(permission);
     }
     throw new AccumuloSecurityException(user, SecurityErrorCode.USER_DOESNT_EXIST); // user
doesn't exist
   }

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/functional/BulkSplitOptimizationTest.java
Tue Jan 31 19:21:02 2012
@@ -23,10 +23,11 @@ import java.util.Map;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.util.CachedConfiguration;
 import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.test.CreateRFiles;
+import org.apache.accumulo.server.test.CreateMapFiles;
 import org.apache.accumulo.server.test.VerifyIngest;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
 
 /**
  * This test verifies that when a lot of files are bulk imported into a table with one tablet
and then splits that not all map files go to the children tablets.
@@ -42,8 +43,8 @@ public class BulkSplitOptimizationTest e
   @Override
   public void cleanup() throws Exception {
     FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
-    fs.delete(new Path("/testrf"), true);
-    fs.delete(new Path("/testrf_failures"), true);
+    fs.delete(new Path("/tmp/testmf"), true);
+    fs.delete(new Path("/tmp/testmf_failures"), true);
   }
   
   @Override
@@ -61,14 +62,14 @@ public class BulkSplitOptimizationTest e
   public void run() throws Exception {
     
     FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
-    fs.delete(new Path("/testrf"), true);
+    fs.delete(new Path("/tmp/testmf"), true);
     
-    CreateRFiles.main(new String[] {"testrf", "8", "0", "100000", "99"});
+    CreateMapFiles.main(new String[] {"tmp/testmf", "8", "0", "100000", "99"});
     
-    bulkImport(fs, TABLE_NAME, "/testrf");
+    bulkImport(fs, TABLE_NAME, "/tmp/testmf");
     
     checkSplits(TABLE_NAME, 0, 0);
-    checkRFiles(TABLE_NAME, 1, 1, 100, 100);
+    checkMapFiles(TABLE_NAME, 1, 1, 100, 100);
     
     // initiate splits
     getConnector().tableOperations().setProperty(TABLE_NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(),
"100K");
@@ -85,6 +86,6 @@ public class BulkSplitOptimizationTest e
     VerifyIngest.main(new String[] {"-timestamp", "1", "-size", "50", "-random", "56", "100000",
"0", "1"});
     
     // ensure each tablet does not have all map files
-    checkRFiles(TABLE_NAME, 50, 100, 1, 4);
+    checkMapFiles(TABLE_NAME, 50, 100, 1, 4);
   }
 }

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/security/TableOp.java
Tue Jan 31 19:21:02 2012
@@ -183,7 +183,9 @@ public class TableOp extends Test {
         }
         for (String s : SecurityHelper.getAuthsArray())
           SecurityHelper.increaseAuthMap(state, s, 1);
-        
+        fs.delete(dir, true);
+        fs.delete(fail, true);
+
         if (!hasPerm)
           throw new AccumuloException("Bulk Import succeeded when it should have failed:
" + dir + " table " + tableName);
         break;

Modified: incubator/accumulo/trunk/test/system/auto/TestUtils.py
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/test/system/auto/TestUtils.py?rev=1238754&r1=1238753&r2=1238754&view=diff
==============================================================================
--- incubator/accumulo/trunk/test/system/auto/TestUtils.py (original)
+++ incubator/accumulo/trunk/test/system/auto/TestUtils.py Tue Jan 31 19:21:02 2012
@@ -41,7 +41,7 @@ FUZZ=os.getpid() % 997
 ACCUMULO_HOME = os.path.dirname(__file__)
 ACCUMULO_HOME = os.path.join(ACCUMULO_HOME, *(os.path.pardir,)*3)
 ACCUMULO_HOME = os.path.realpath(ACCUMULO_HOME)
-ACCUMULO_DIR = "/accumulo-" + ID
+ACCUMULO_DIR = "/user/" + os.getlogin() + "/accumulo-" + ID
 SITE = "test-" + ID
 
 WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)



Mime
View raw message