accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1342779 - in /accumulo/branches/ACCUMULO-578: ./ core/ core/src/main/java/org/apache/accumulo/core/conf/ core/src/main/java/org/apache/accumulo/core/util/shell/commands/ server/ server/src/main/java/org/apache/accumulo/server/tabletserver/...
Date Fri, 25 May 2012 19:05:30 GMT
Author: ecn
Date: Fri May 25 19:05:29 2012
New Revision: 1342779

URL: http://svn.apache.org/viewvc?rev=1342779&view=rev
Log:
ACCUMULO-578 merged most recent trunk changes

Added:
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
      - copied unchanged from r1342773, accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
Modified:
    accumulo/branches/ACCUMULO-578/   (props changed)
    accumulo/branches/ACCUMULO-578/core/   (props changed)
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
    accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
    accumulo/branches/ACCUMULO-578/server/   (props changed)
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
    accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/functional/BloomFilterTest.java
    accumulo/branches/ACCUMULO-578/src/   (props changed)
    accumulo/branches/ACCUMULO-578/test/system/auto/TestUtils.py
    accumulo/branches/ACCUMULO-578/test/system/auto/simple/aggregation.py
    accumulo/branches/ACCUMULO-578/test/system/auto/stress/bigRootTablet.py
    accumulo/branches/ACCUMULO-578/test/system/auto/stress/weird.py
    accumulo/branches/ACCUMULO-578/test/system/randomwalk/conf/modules/Shard.xml

Propchange: accumulo/branches/ACCUMULO-578/
------------------------------------------------------------------------------
  Merged /accumulo/trunk:r1342237-1342773
  Merged /accumulo/branches/1.4:r1342419

Propchange: accumulo/branches/ACCUMULO-578/core/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/core:r1342237-1342773
  Merged /accumulo/branches/1.4/core:r1342419

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Fri May 25 19:05:29 2012
@@ -56,7 +56,7 @@ public enum Property {
   // properties that are specific to master server behavior
   MASTER_PREFIX("master.", null, PropertyType.PREFIX, "Properties in this category affect
the behavior of the master server"),
   MASTER_CLIENTPORT("master.port.client", "9999", PropertyType.PORT, "The port used for handling
client connections on the master"),
-  MASTER_TABLET_BALANCER("master.tablet.balancer", "org.apache.accumulo.server.master.balancer.DefaultLoadBalancer",
PropertyType.CLASSNAME,
+  MASTER_TABLET_BALANCER("master.tablet.balancer", "org.apache.accumulo.server.master.balancer.TableLoadBalancer",
PropertyType.CLASSNAME,
       "The balancer class that accumulo will use to make tablet assignment and migration
decisions."),
   MASTER_RECOVERY_MAXAGE("master.recovery.max.age", "60m", PropertyType.TIMEDURATION, "Recovery
files older than this age will be removed."),
   MASTER_RECOVERY_MAXTIME("master.recovery.time.max", "30m", PropertyType.TIMEDURATION, "The
maximum time to attempt recovery before giving up"),

Modified: accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
(original)
+++ accumulo/branches/ACCUMULO-578/core/src/main/java/org/apache/accumulo/core/util/shell/commands/SetIterCommand.java
Fri May 25 19:05:29 2012
@@ -29,10 +29,12 @@ import org.apache.accumulo.core.client.A
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.iterators.AggregatingIterator;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.OptionDescriber;
 import org.apache.accumulo.core.iterators.OptionDescriber.IteratorOptions;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.aggregation.Aggregator;
 import org.apache.accumulo.core.iterators.user.AgeOffFilter;
 import org.apache.accumulo.core.iterators.user.RegExFilter;
 import org.apache.accumulo.core.iterators.user.ReqVisFilter;
@@ -50,7 +52,7 @@ import org.apache.commons.cli.Options;
 public class SetIterCommand extends Command {
   
   private Option mincScopeOpt, majcScopeOpt, scanScopeOpt, nameOpt, priorityOpt;
-  private Option ageoffTypeOpt, regexTypeOpt, versionTypeOpt, reqvisTypeOpt, classnameTypeOpt;
+  private Option aggTypeOpt, ageoffTypeOpt, regexTypeOpt, versionTypeOpt, reqvisTypeOpt,
classnameTypeOpt;
   
   public int execute(String fullCommand, CommandLine cl, Shell shellState) throws AccumuloException,
AccumuloSecurityException, TableNotFoundException,
       IOException, ShellCommandException {
@@ -60,7 +62,10 @@ public class SetIterCommand extends Comm
     
     Map<String,String> options = new HashMap<String,String>();
     String classname = cl.getOptionValue(classnameTypeOpt.getOpt());
-    if (cl.hasOption(regexTypeOpt.getOpt()))
+    if (cl.hasOption(aggTypeOpt.getOpt())) {
+      Shell.log.warn("aggregators are deprecated");
+      classname = AggregatingIterator.class.getName();
+    } else if (cl.hasOption(regexTypeOpt.getOpt()))
       classname = RegExFilter.class.getName();
     else if (cl.hasOption(ageoffTypeOpt.getOpt()))
       classname = AgeOffFilter.class.getName();
@@ -75,6 +80,11 @@ public class SetIterCommand extends Comm
     
     String name = cl.getOptionValue(nameOpt.getOpt(), setUpOptions(shellState.getReader(),
classname, options));
     
+    String aggregatorClass = options.get("aggregatorClass");
+    if (aggregatorClass != null && !shellState.getConnector().instanceOperations().testClassLoad(aggregatorClass,
Aggregator.class.getName()))
+      throw new ShellCommandException(ErrorCode.INITIALIZATION_FAILURE, "Servers are unable
to load " + aggregatorClass + " as type "
+          + Aggregator.class.getName());
+    
     setTableProperties(cl, shellState, tableName, priority, options, classname, name);
     
     return 0;
@@ -207,12 +217,14 @@ public class SetIterCommand extends Comm
     OptionGroup typeGroup = new OptionGroup();
     classnameTypeOpt = new Option("class", "class-name", true, "a java class that implements
SortedKeyValueIterator");
     classnameTypeOpt.setArgName("name");
+    aggTypeOpt = new Option("agg", "aggregator", false, "an aggregating type");
     regexTypeOpt = new Option("regex", "regular-expression", false, "a regex matching iterator");
     versionTypeOpt = new Option("vers", "version", false, "a versioning iterator");
     reqvisTypeOpt = new Option("reqvis", "require-visibility", false, "an iterator that omits
entries with empty visibilities");
     ageoffTypeOpt = new Option("ageoff", "ageoff", false, "an aging off iterator");
     
     typeGroup.addOption(classnameTypeOpt);
+    typeGroup.addOption(aggTypeOpt);
     typeGroup.addOption(regexTypeOpt);
     typeGroup.addOption(versionTypeOpt);
     typeGroup.addOption(reqvisTypeOpt);

Propchange: accumulo/branches/ACCUMULO-578/server/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.4/server:r1342419
  Merged /accumulo/trunk/server:r1342237-1342773

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Fri May 25 19:05:29 2012
@@ -2214,6 +2214,12 @@ public class Tablet {
         this.stats = minorCompact(conf, fs, tabletMemory.getMinCMemTable(), newMapfileLocation
+ "_tmp", newMapfileLocation, mergeFile, true, queued,
             commitSession, flushId);
         span.stop();
+        
+        if (needsSplit()) {
+          tabletServer.executeSplit(Tablet.this);
+        } else {
+          initiateMajorCompaction(MajorCompactionReason.NORMAL);
+        }
       } catch (Throwable t) {
         log.error("Unknown error during minor compaction for extent: " + getExtent(), t);
         throw new RuntimeException(t);
@@ -3229,7 +3235,8 @@ public class Tablet {
           span.data("written", "" + mcs.getEntriesWritten());
           majCStats.add(mcs);
           
-          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
filesToCompact.size() == 0 ? compactionId : null,
+          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
+              filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst()
: null,
               new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
           
           // when major compaction produces a file w/ zero entries, it will be deleted...
do not want
@@ -3602,6 +3609,12 @@ public class Tablet {
     try {
       datafileManager.importMapFiles(tid, entries, setTime);
       lastMapFileImportTime = System.currentTimeMillis();
+      
+      if (needsSplit()) {
+        tabletServer.executeSplit(this);
+      } else {
+        initiateMajorCompaction(MajorCompactionReason.NORMAL);
+      }
     } finally {
       synchronized (this) {
         if (writesInProgress < 1)

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Fri May 25 19:05:29 2012
@@ -2131,6 +2131,10 @@ public class TabletServer extends Abstra
     return majorCompactorDisabled;
   }
   
+  void executeSplit(Tablet tablet) {
+    resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new SplitRunner(tablet)));
+  }
+
   private class MajorCompactor implements Runnable {
     
     public void run() {
@@ -2164,7 +2168,7 @@ public class TabletServer extends Abstra
             // if we need to split AND compact, we need a good way
             // to decide what to do
             if (tablet.needsSplit()) {
-              resourceManager.executeSplit(tablet.getExtent(), new LoggingRunnable(log, new
SplitRunner(tablet)));
+              executeSplit(tablet);
               continue;
             }
             
@@ -3388,7 +3392,6 @@ public class TabletServer extends Abstra
   public TableConfiguration getTableConfiguration(KeyExtent extent) {
     return ServerConfiguration.getTableConfiguration(instance, extent.getTableId().toString());
   }
-
   public DfsLogger.ServerConfig getServerConfig() {
     return new DfsLogger.ServerConfig() {
       

Modified: accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/functional/BloomFilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/functional/BloomFilterTest.java?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/functional/BloomFilterTest.java
(original)
+++ accumulo/branches/ACCUMULO-578/server/src/main/java/org/apache/accumulo/server/test/functional/BloomFilterTest.java
Fri May 25 19:05:29 2012
@@ -161,6 +161,8 @@ public class BloomFilterTest extends Fun
       throw new Exception("Queries exceeded expected run time " + (t2 - t1) / 1000.0 + "
" + secs);
     }
     
+    bs.close();
+
   }
   
   private void write(String table, int depth, long start, long end, int step) throws Exception
{

Propchange: accumulo/branches/ACCUMULO-578/src/
------------------------------------------------------------------------------
  Merged /accumulo/trunk/src:r1342237-1342773
  Merged /accumulo/branches/1.4/src:r1342419

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/TestUtils.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/TestUtils.py?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/TestUtils.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/TestUtils.py Fri May 25 19:05:29 2012
@@ -47,6 +47,7 @@ SITE = "test-" + ID
 WALOG = os.path.join(ACCUMULO_HOME, 'walogs', ID)
 LOG_PROPERTIES= os.path.join(ACCUMULO_HOME, 'conf', 'log4j.properties')
 LOG_GENERIC = os.path.join(ACCUMULO_HOME, 'conf', 'generic_logger.xml')
+LOG_MONITOR = os.path.join(ACCUMULO_HOME, 'conf', 'monitor_logger.xml')
 General_CLASSPATH = ("$ACCUMULO_HOME/lib/[^.].$ACCUMULO_VERSION.jar, $ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,"
 "$HADOOP_HOME/conf,$HADOOP_HOME/[^.].*.jar, $HADOOP_HOME/lib/[^.].*.jar") 
 
@@ -281,9 +282,12 @@ class TestUtilsMixin:
          os.rename(LOG_PROPERTIES, '%s.bkp' % LOG_PROPERTIES)
       if os.path.exists(LOG_GENERIC):
          os.rename(LOG_GENERIC, '%s.bkp' % LOG_GENERIC)
+      if os.path.exists(LOG_MONITOR):
+         os.rename(LOG_MONITOR, '%s.bkp' % LOG_MONITOR)
       
       shutil.copyfile('%s/conf/examples/512MB/standalone/log4j.properties' % ACCUMULO_HOME,
LOG_PROPERTIES)
       shutil.copyfile('%s/conf/examples/512MB/standalone/generic_logger.xml' % ACCUMULO_HOME,
LOG_GENERIC)
+      shutil.copyfile('%s/conf/examples/512MB/standalone/monitor_logger.xml' % ACCUMULO_HOME,
LOG_MONITOR)
       
 
     def start_accumulo_procs(self, safeMode=None):
@@ -396,12 +400,16 @@ class TestUtilsMixin:
     def clean_logging(self):
       LOG_PROPERTIES_BACKUP='%s.bkp' % LOG_PROPERTIES 
       LOG_GENERIC_BACKUP='%s.bkp' % LOG_GENERIC
+      LOG_MONITOR_BACKUP='%s.bkp' % LOG_MONITOR
       os.remove(LOG_PROPERTIES)
       os.remove(LOG_GENERIC)
+      os.remove(LOG_MONITOR)
       if os.path.exists(LOG_PROPERTIES_BACKUP):
         os.rename(LOG_PROPERTIES_BACKUP, LOG_PROPERTIES)
       if os.path.exists(LOG_GENERIC_BACKUP):
          os.rename(LOG_GENERIC_BACKUP, LOG_GENERIC)
+      if os.path.exists(LOG_MONITOR_BACKUP):
+         os.rename(LOG_MONITOR_BACKUP, LOG_MONITOR)
 
     def sleep(self, secs):
         log.debug("Sleeping %f seconds" % secs)

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/simple/aggregation.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/simple/aggregation.py?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/simple/aggregation.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/simple/aggregation.py Fri May 25 19:05:29
2012
@@ -43,7 +43,7 @@ class AggregationTest(TestUtilsMixin, un
 
         # initialize the database
         aggregator = 'org.apache.accumulo.core.iterators.aggregation.StringSummation'
-        cmd = 'createtable test -a cf=' + aggregator
+        cmd = 'createtable test\nsetiter -agg -minc -majc -scan -p 10 -t test\ncf ' + aggregator
+ '\n\n'
         out, err, code = self.rootShell(self.masterHost(),"%s\n" % cmd)
         self.assert_(code == 0)
 

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/stress/bigRootTablet.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/stress/bigRootTablet.py?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/stress/bigRootTablet.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/stress/bigRootTablet.py Fri May 25 19:05:29
2012
@@ -21,7 +21,7 @@ from TestUtils import TestUtilsMixin
 
 log = logging.getLogger('test.auto')
 
-class BigRootTablet(unittest.TestCase, TestUtilsMixin):
+class BigRootTablet(TestUtilsMixin, unittest.TestCase):
     "ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver
scan buffers"
 
     order = 80
@@ -33,6 +33,9 @@ class BigRootTablet(unittest.TestCase, T
     def setUp(self):
         TestUtilsMixin.setUp(self);
     
+    def tearDown(self):
+        TestUtilsMixin.tearDown(self);
+    
     def runTest(self):
 	cmd = 'table !METADATA\naddsplits 0 1 2 3 4 5 6 7 8 9 a\n'
         for i in range(10):

Modified: accumulo/branches/ACCUMULO-578/test/system/auto/stress/weird.py
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/auto/stress/weird.py?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/auto/stress/weird.py (original)
+++ accumulo/branches/ACCUMULO-578/test/system/auto/stress/weird.py Fri May 25 19:05:29 2012
@@ -22,7 +22,7 @@ from TestUtils import TestUtilsMixin
 
 log = logging.getLogger('test.auto')
 
-class LateLastContact(unittest.TestCase, TestUtilsMixin):
+class LateLastContact(TestUtilsMixin, unittest.TestCase):
     """Fake the "tablet stops talking but holds its lock" problem we see when hard drives
and NFS fail.
        Start a ZombieTServer, and see that master stops it.
     """
@@ -35,6 +35,9 @@ class LateLastContact(unittest.TestCase,
     def setUp(self):
         TestUtilsMixin.setUp(self);
     
+    def tearDown(self):
+        TestUtilsMixin.tearDown(self);
+
     def runTest(self):
         handle = self.runClassOn(self.masterHost(), 'org.apache.accumulo.server.test.functional.ZombieTServer',
[])
         out, err = handle.communicate()

Modified: accumulo/branches/ACCUMULO-578/test/system/randomwalk/conf/modules/Shard.xml
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-578/test/system/randomwalk/conf/modules/Shard.xml?rev=1342779&r1=1342778&r2=1342779&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-578/test/system/randomwalk/conf/modules/Shard.xml (original)
+++ accumulo/branches/ACCUMULO-578/test/system/randomwalk/conf/modules/Shard.xml Fri May 25
19:05:29 2012
@@ -94,6 +94,11 @@
   <edge id="dummy.ToAll" weight="1"/>
 </node>
 
+<node id="shard.CompactFilter">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+
 <node id="dummy.ToAll">
   <edge id="shard.Reindex" weight="3"/>
   <edge id="shard.Flush" weight="3"/>
@@ -102,11 +107,12 @@
   <edge id="shard.Split" weight="40"/>
   <edge id="shard.Merge" weight="20"/>
   <edge id="shard.DeleteWord" weight="9"/>
+  <edge id="shard.CompactFilter" weight="9"/>
   <edge id="shard.DeleteSomeDocs" weight="20"/>
   <edge id="shard.BulkInsert" weight="3"/>
   <edge id="shard.Delete" weight="486"/>
-  <edge id="shard.Insert" weight="4697"/>
-  <edge id="shard.Search" weight="4696"/>
+  <edge id="shard.Insert" weight="4693"/>
+  <edge id="shard.Search" weight="4691"/>
 </node>
 
 </module>



Mime
View raw message