accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [18/18] git commit: ACCUMULO-1957 backed out changes I didn't mean to commit, added a test of the durability option for InsertCommand
Date Fri, 05 Sep 2014 21:17:34 GMT
ACCUMULO-1957 backed out changes I didn't mean to commit, added a test of the durability option
for InsertCommand


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6e02b3c7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6e02b3c7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6e02b3c7

Branch: refs/heads/master
Commit: 6e02b3c7bd96c7ab3840f311bfc62f5727549b75
Parents: 3a0beab
Author: Eric C. Newton <eric.newton@gmail.com>
Authored: Thu Sep 4 15:07:36 2014 -0400
Committer: Eric C. Newton <eric.newton@gmail.com>
Committed: Fri Sep 5 17:16:59 2014 -0400

----------------------------------------------------------------------
 .../java/org/apache/accumulo/core/conf/Property.java  |  6 +-----
 .../examples/simple/client/ReadWriteExample.java      |  2 +-
 .../org/apache/accumulo/tserver/TabletServer.java     | 14 ++------------
 .../accumulo/tserver/TabletServerResourceManager.java |  2 +-
 .../org/apache/accumulo/tserver/tablet/Tablet.java    |  1 -
 .../apache/accumulo/shell/commands/InsertCommand.java |  3 ++-
 .../accumulo/test/randomwalk/concurrent/Config.java   |  2 +-
 .../java/org/apache/accumulo/proxy/SimpleProxyIT.java | 12 ++++++------
 .../java/org/apache/accumulo/test/ShellServerIT.java  | 10 ++++++++++
 .../accumulo/test/functional/BloomFilterIT.java       |  2 +-
 10 files changed, 25 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 35cd0a6..5401c7c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -197,15 +197,11 @@ public enum Property {
   TSERV_INDEXCACHE_SIZE("tserver.cache.index.size", "512M", PropertyType.MEMORY, "Specifies
the size of the cache for file indices."),
   TSERV_PORTSEARCH("tserver.port.search", "false", PropertyType.BOOLEAN, "if the ports above
are in use, search higher ports until one is available"),
   TSERV_CLIENTPORT("tserver.port.client", "9997", PropertyType.PORT, "The port used for handling
client connections on the tablet servers"),
-  @Deprecated
   TSERV_MUTATION_QUEUE_MAX("tserver.mutation.queue.max", "1M", PropertyType.MEMORY,
-      "This setting is deprecated. See tserver.total.mutation.queue.max. " 
-          + "The amount of memory to use to store write-ahead-log mutations-per-session before
flushing them. Since the buffer is per write session, consider the"
+          "The amount of memory to use to store write-ahead-log mutations-per-session before
flushing them. Since the buffer is per write session, consider the"
           + " max number of concurrent writer when configuring. When using Hadoop 2, Accumulo
will call hsync() on the WAL . For a small number of "
           + "concurrent writers, increasing this buffer size decreases the frequncy of hsync
calls. For a large number of concurrent writers a small buffers "
           + "size is ok because of group commit."),
-  TSERV_TOTAL_MUTATION_QUEUE_MAX("tserver.total.mutation.queue.max", "50M", PropertyType.MEMORY,

-      "The amount of memory used to store write-ahead-log mutations before flushing them."),
   TSERV_TABLET_SPLIT_FINDMIDPOINT_MAXOPEN("tserver.tablet.split.midpoint.files.max", "30",
PropertyType.COUNT,
       "To find a tablets split points, all index files are opened. This setting determines
how many index "
           + "files can be opened at once. When there are more index files than this setting
multiple passes "

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
index 2a42d29..ccc924b 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/client/ReadWriteExample.java
@@ -64,7 +64,7 @@ public class ReadWriteExample {
     boolean readEntries = false;
     @Parameter(names = {"-d", "--delete"}, description = "delete entries after any creates")
     boolean deleteEntries = false;
-    @Parameter(names = {"--durability"}, description = "durabilty used for writes (none,
log, flush or sync)", converter=DurabilityConverter.class)
+    @Parameter(names = {"--durability"}, description = "durability used for writes (none,
log, flush or sync)", converter=DurabilityConverter.class)
     Durability durability = Durability.DEFAULT;
 
     public Opts() {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index f164621..dc9f27f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -351,8 +351,6 @@ public class TabletServer implements Runnable {
 
   private final RowLocks rowLocks = new RowLocks();
   
-  private final AtomicLong totalQueuedMutationSize = new AtomicLong(0);
-
   private class ThriftClientHandler extends ClientServiceHandler implements TabletClientService.Iface
{
 
     ThriftClientHandler() {
@@ -729,16 +727,13 @@ public class TabletServer implements Runnable {
         setUpdateTablet(us, keyExtent);
 
         if (us.currentTablet != null) {
-          long additionalMutationSize = 0;
           List<Mutation> mutations = us.queuedMutations.get(us.currentTablet);
           for (TMutation tmutation : tmutations) {
             Mutation mutation = new ServerMutation(tmutation);
             mutations.add(mutation);
-            additionalMutationSize += mutation.numBytes();
+            us.queuedMutationSize += mutation.numBytes();
           }
-          us.queuedMutationSize += additionalMutationSize;
-          long totalQueued = TabletServer.this.updateTotalQueuedMutationSize(additionalMutationSize);
-          if (totalQueued > TabletServer.this.getConfiguration().getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX))
{
+          if (us.queuedMutationSize > TabletServer.this.getConfiguration().getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX))
{
             flush(us);
           }
         }
@@ -887,7 +882,6 @@ public class TabletServer implements Runnable {
         if (us.currentTablet != null) {
           us.queuedMutations.put(us.currentTablet, new ArrayList<Mutation>());
         }
-        TabletServer.this.updateTotalQueuedMutationSize(-us.queuedMutationSize);
         us.queuedMutationSize = 0;
       }
       us.totalUpdates += mutationCount;
@@ -1764,10 +1758,6 @@ public class TabletServer implements Runnable {
     return majorCompactorDisabled;
   }
 
-  public long updateTotalQueuedMutationSize(long additionalMutationSize) {
-    return totalQueuedMutationSize .addAndGet(additionalMutationSize);
-  }
-
   public Tablet getOnlineTablet(KeyExtent extent) {
     return onlineTablets.get(extent);
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index 25c0ee8..3d42c7c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -156,7 +156,7 @@ public class TabletServerResourceManager {
     long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
     long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
     long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
-    long totalQueueSize = acuConf.getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX);
+    long totalQueueSize = acuConf.getMemoryInBytes(Property.TSERV_MUTATION_QUEUE_MAX);
 
     _iCache = new LruBlockCache(iCacheSize, blockSize);
     _dCache = new LruBlockCache(dCacheSize, blockSize);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index c3380bf..1151390 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2385,7 +2385,6 @@ public class Tablet implements TabletCommitter {
 
   // this lock is basically used to synchronize writing of log info to metadata
   private final ReentrantLock logLock = new ReentrantLock();
-  private AtomicLong totalQueuedMutationSize = new AtomicLong(0);
 
   public synchronized int getLogCount() {
     return currentLogs.size();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/shell/src/main/java/org/apache/accumulo/shell/commands/InsertCommand.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/InsertCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/InsertCommand.java
index 5fae7ec..16afc9e 100644
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/InsertCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/InsertCommand.java
@@ -85,7 +85,7 @@ public class InsertCommand extends Command {
       String userDurability = cl.getOptionValue(durabilityOption.getOpt());
       switch (userDurability) {
         case "sync":
-          cfg.setDurability(Durability.SYNC); 
+          cfg.setDurability(Durability.SYNC);
           break;
         case "flush":
           cfg.setDurability(Durability.FLUSH);
@@ -160,6 +160,7 @@ public class InsertCommand extends Command {
     o.addOption(timeoutOption);
     
     durabilityOption = new Option("d", "durability", true, "durability to use for insert,
should be one of \"none\" \"log\" \"flush\" or \"sync\"");
+    o.addOption(durabilityOption);
     
     return o;
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
index 8d14574..4af85a7 100644
--- a/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
+++ b/test/src/main/java/org/apache/accumulo/test/randomwalk/concurrent/Config.java
@@ -72,7 +72,7 @@ public class Config extends Test {
       s(Property.TSERV_MAXMEM, 1000000, 3 * 1024 * 1024 * 1024L),
       s(Property.TSERV_READ_AHEAD_MAXCONCURRENT, 1, 25),
       s(Property.TSERV_MIGRATE_MAXCONCURRENT, 1, 10),
-      s(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, 10000, 1024 * 1024),
+      s(Property.TSERV_MUTATION_QUEUE_MAX, 10000, 1024 * 1024),
       s(Property.TSERV_RECOVERY_MAX_CONCURRENT, 1, 100),
       s(Property.TSERV_SCAN_MAX_OPENFILES, 10, 1000),
       s(Property.TSERV_THREADCHECK, 100, 10000),

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/test/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java b/test/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
index 62e5b64..6b4bcfb 100644
--- a/test/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
+++ b/test/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java
@@ -632,7 +632,7 @@ public class SimpleProxyIT {
       client.createConditionalWriter(creds, doesNotExist, new ConditionalWriterOptions());
     } catch (TableNotFoundException ex) {}
   }
-  
+
   @Test(timeout = 10 * 1000)
   public void testExists() throws Exception {
     client.createTable(creds, "ett1", false, TimeType.MILLIS);
@@ -806,7 +806,7 @@ public class SimpleProxyIT {
     };
     t.start();
 
-    // look for the scan many times 
+    // look for the scan many times
     List<ActiveScan> scans = new ArrayList<ActiveScan>();
     for (int i = 0; i < 100 && scans.isEmpty(); i++) {
       for (String tserver : client.getTabletServers(creds)) {
@@ -1012,7 +1012,7 @@ public class SimpleProxyIT {
     assertScan(new String[][] {}, TABLE_TEST);
 
     UtilWaitThread.sleep(2000);
-    
+
     writerOptions = new WriterOptions();
     writerOptions.setLatencyMs(10000);
     writerOptions.setMaxMemory(3000);
@@ -1551,11 +1551,11 @@ public class SimpleProxyIT {
   static private ByteBuffer t2bb(Text t) {
     return ByteBuffer.wrap(t.getBytes());
   }
-  
+
   @Test
   public void testGetRowRange() throws Exception {
     Range range = client.getRowRange(s2bb("xyzzy"));
-    org.apache.accumulo.core.data.Range range2 = new org.apache.accumulo.core.data.Range(new
Text("xyzzy")); 
+    org.apache.accumulo.core.data.Range range2 = new org.apache.accumulo.core.data.Range(new
Text("xyzzy"));
     assertEquals(0, range.start.row.compareTo(t2bb(range2.getStartKey().getRow())));
     assertEquals(0, range.stop.row.compareTo(t2bb(range2.getEndKey().getRow())));
     assertEquals(range.startInclusive, range2.isStartKeyInclusive());
@@ -1567,7 +1567,7 @@ public class SimpleProxyIT {
     assertEquals(range.start.timestamp, range.start.timestamp);
     assertEquals(range.stop.timestamp, range.stop.timestamp);
   }
-  
+
   @AfterClass
   public static void tearDownMiniCluster() throws Exception {
     accumulo.stop();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
index 04fdd1c..b1ad3d8 100644
--- a/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ShellServerIT.java
@@ -421,6 +421,16 @@ public class ShellServerIT extends SimpleMacIT {
     ts.exec("deleteuser -f xyzzy", true);
     ts.exec("users", true, "xyzzy", false);
   }
+  
+  @Test(timeout = 60 * 1000)
+  public void durability() throws Exception {
+    final String table = name.getMethodName();
+    ts.exec("createtable " + table);
+    ts.exec("insert -d none a cf cq randomGunkaASDFWEAQRd");
+    ts.exec("insert -d foo a cf cq2 2", false, "foo", true);
+    ts.exec("scan -r a", true, "randomGunkaASDFWEAQRd", true);
+    ts.exec("scan -r a", true, "foo", false);
+  }  
 
   @Test(timeout = 45000)
   public void iter() throws Exception {

http://git-wip-us.apache.org/repos/asf/accumulo/blob/6e02b3c7/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
index 6ee671e..8f6b830 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java
@@ -57,7 +57,7 @@ public class BloomFilterIT extends ConfigurableMacIT {
     siteConfig.put(Property.TABLE_BLOOM_SIZE.getKey(), "2000000");
     siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%");
     siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0");
-    siteConfig.put(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX.getKey(), "10M");
+    siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M");
     siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64K");
     cfg.setSiteConfig(siteConfig );
   }


Mime
View raw message