incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1295272 - in /incubator/accumulo/trunk: ./ src/core/ src/packages/deb/accumulo-native/ src/packages/deb/accumulo/ src/server/ src/server/src/main/java/org/apache/accumulo/server/master/ src/server/src/main/java/org/apache/accumulo/server/m...
Date Wed, 29 Feb 2012 21:05:59 GMT
Author: kturner
Date: Wed Feb 29 21:05:58 2012
New Revision: 1295272

URL: http://svn.apache.org/viewvc?rev=1295272&view=rev
Log:
ACCUMULO-436 ACCUMULO-19 ACCUMULO-431 merge from 1.4

Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/   (props changed)
    incubator/accumulo/trunk/src/packages/deb/accumulo-native/control
    incubator/accumulo/trunk/src/packages/deb/accumulo/control
    incubator/accumulo/trunk/src/server/   (props changed)
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
    incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
    incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml
    incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195,1230180,1230736,1231043,1236873,1245632
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1294846
+/incubator/accumulo/branches/1.4:1201902-1295271

Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
 /incubator/accumulo/branches/1.3.5rc/src/core:1209938
-/incubator/accumulo/branches/1.4/src/core:1201902-1294805
+/incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
+/incubator/accumulo/branches/1.4/src/core:1201902-1295271

Modified: incubator/accumulo/trunk/src/packages/deb/accumulo-native/control
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/packages/deb/accumulo-native/control?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/packages/deb/accumulo-native/control (original)
+++ incubator/accumulo/trunk/src/packages/deb/accumulo-native/control Wed Feb 29 21:05:58
2012
@@ -21,6 +21,6 @@ Package: accumulo-native
 Version: [[version]]
 Section: misc
 Architecture: all
-Depends: default-jdk, accumulo (=[[version]]), g++, g++-multilib
+Depends: java6-sdk, accumulo (=[[version]]), g++, g++-multilib
 Description: Native libraries for Accumulo- locks and InMemoryMap
   This optional package contains native libraries that improve the performance of Accumulo
tservers.

Modified: incubator/accumulo/trunk/src/packages/deb/accumulo/control
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/packages/deb/accumulo/control?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/packages/deb/accumulo/control (original)
+++ incubator/accumulo/trunk/src/packages/deb/accumulo/control Wed Feb 29 21:05:58 2012
@@ -21,7 +21,7 @@ Package: accumulo
 Version: [[version]]
 Section: misc
 Architecture: all
-Depends: default-jre-headless, hadoop, hadoop-zookeeper, coreutils
+Depends: java6-runtime, hadoop, hadoop-zookeeper, coreutils, ssh-server
 Recommends: accumulo-native
 Description: Accumulo is a large distributed structured store based on Google's BigTable
design.
 Distribution: development

Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 29 21:05:58 2012
@@ -1,3 +1,3 @@
-/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
 /incubator/accumulo/branches/1.3.5rc/src/server:1209938
-/incubator/accumulo/branches/1.4/src/server:1201902-1294805
+/incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
+/incubator/accumulo/branches/1.4/src/server:1201902-1295271

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Wed Feb 29 21:05:58 2012
@@ -41,11 +41,11 @@ import java.util.concurrent.atomic.Atomi
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchDeleter;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.Connector;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IsolatedScanner;
+import org.apache.accumulo.core.client.MutationsRejectedException;
 import org.apache.accumulo.core.client.RowIterator;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
@@ -1506,26 +1506,30 @@ public class Master implements LiveTServ
     private void updateMergeState(Map<Text,MergeStats> mergeStatsCache) {
       for (MergeStats stats : mergeStatsCache.values()) {
         try {
-          MergeState update = stats.nextMergeState();
-          if (update == MergeState.MERGING) {
-            if (stats.verifyMergeConsistency(getConnector(), Master.this)) {
-              try {
-                if (stats.getMergeInfo().isDelete()) {
-                  deleteTablets(stats.getMergeInfo());
-                } else {
-                  mergeMetadataRecords(stats.getMergeInfo());
-                }
-                setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
-              } catch (Exception ex) {
-                log.error("Unable merge metadata table records", ex);
-              }
-            }
-          }
+          MergeState update = stats.nextMergeState(getConnector(), Master.this);
+          
+          // when next state is MERGING, its important to persist this before
+          // starting the merge... the verification check that is done before
+          // moving into the merging state could fail if merge starts but does
+          // not finish
           if (update == MergeState.COMPLETE)
             update = MergeState.NONE;
           if (update != stats.getMergeInfo().getState()) {
             setMergeState(stats.getMergeInfo(), update);
           }
+
+          if (update == MergeState.MERGING) {
+            try {
+              if (stats.getMergeInfo().isDelete()) {
+                deleteTablets(stats.getMergeInfo());
+              } else {
+                mergeMetadataRecords(stats.getMergeInfo());
+              }
+              setMergeState(stats.getMergeInfo(), update = MergeState.COMPLETE);
+            } catch (Exception ex) {
+              log.error("Unable merge metadata table records", ex);
+            }
+          }
         } catch (Exception ex) {
           log.error("Unable to update merge state for merge " + stats.getMergeInfo().getRange(),
ex);
         }
@@ -1579,18 +1583,21 @@ public class Master implements LiveTServ
           }
         }
         MetadataTable.addDeleteEntries(range, datafiles, SecurityConstants.getSystemCredentials());
-        log.debug("Removing metadata table entries in range " + deleteRange);
-        BatchDeleter bd = conn.createBatchDeleter(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS,
4, 100000l, 1000l, 4);
-        bd.setRanges(Collections.singleton(deleteRange));
-        bd.delete();
-        bd.close();
         
+        BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000000l,
100l, 1);
+        try {
+          deleteTablets(deleteRange, bw, conn);
+        } finally {
+          bw.close();
+        }
+
         if (followingTablet != null) {
           log.debug("Updating prevRow of " + followingTablet + " to " + range.getPrevEndRow());
-          BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000l, 100l,
1);
+          bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, 1000l, 100l, 1);
           try {
             Mutation m = new Mutation(followingTablet.getMetadataEntry());
             ColumnFQ.put(m, Constants.METADATA_PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(range.getPrevEndRow()));
+            ColumnFQ.putDelete(m, Constants.METADATA_CHOPPED_COLUMN);
             bw.addMutation(m);
             bw.flush();
           } finally {
@@ -1618,7 +1625,7 @@ public class Master implements LiveTServ
       if (start == null) {
         start = new Text();
       }
-      Range scanRange = new Range(KeyExtent.getMetadataEntry(range.getTableId(), start),
false, stopRow, true);
+      Range scanRange = new Range(KeyExtent.getMetadataEntry(range.getTableId(), start),
false, stopRow, false);
       BatchWriter bw = null;
       try {
         long fileCount = 0;
@@ -1629,14 +1636,13 @@ public class Master implements LiveTServ
         scanner.setRange(scanRange);
         ColumnFQ.fetch(scanner, Constants.METADATA_PREV_ROW_COLUMN);
         ColumnFQ.fetch(scanner, Constants.METADATA_TIME_COLUMN);
+        ColumnFQ.fetch(scanner, Constants.METADATA_DIRECTORY_COLUMN);
         scanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
         Mutation m = new Mutation(stopRow);
         String maxLogicalTime = null;
         for (Entry<Key,Value> entry : scanner) {
           Key key = entry.getKey();
           Value value = entry.getValue();
-          if (key.getRow().equals(stopRow))
-            break;
           if (key.getColumnFamily().equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
             m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
             fileCount++;
@@ -1645,6 +1651,8 @@ public class Master implements LiveTServ
             firstPrevRowValue = new Value(value);
           } else if (Constants.METADATA_TIME_COLUMN.hasColumns(key)) {
             maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, value.toString());
+          } else if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
+            bw.addMutation(MetadataTable.createDeleteMutation(range.getTableId().toString(),
entry.getValue().toString()));
           }
         }
         
@@ -1663,8 +1671,10 @@ public class Master implements LiveTServ
         
         if (!m.getUpdates().isEmpty()) {
           bw.addMutation(m);
-          bw.flush();
         }
+        
+        bw.flush();
+
         log.debug("Moved " + fileCount + " files to " + stop);
         
         if (firstPrevRowValue == null) {
@@ -1677,38 +1687,13 @@ public class Master implements LiveTServ
         log.debug("Setting the prevRow for last tablet: " + stop);
         bw.addMutation(updatePrevRow);
         bw.flush();
-        
-        // Delete everything in the other tablets
-        scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
-        log.debug("Scanning range " + scanRange);
-        scanner.setRange(scanRange);
-        for (Entry<Key,Value> entry : scanner) {
-          Key key = entry.getKey();
-          if (key.getRow().equals(stopRow))
-            break;
-          if (Constants.METADATA_DIRECTORY_COLUMN.hasColumns(key)) {
-            bw.addMutation(MetadataTable.createDeleteMutation(range.getTableId().toString(),
entry.getValue().toString()));
-          }
-          
-          // TODO could group by row
-          m = new Mutation(key.getRow());
-          m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
-          log.debug("deleting entry " + key);
-          bw.addMutation(m);
-        }
-        bw.flush();
+
+        deleteTablets(scanRange, bw, conn);
         
         // Clean-up the last chopped marker
-        scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
-        scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
-        scanner.setRange(new Range(stopRow, stopRow));
-        for (Entry<Key,Value> entry : scanner) {
-          Key key = entry.getKey();
-          m = new Mutation(key.getRow());
-          m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
-          log.debug("deleting entry " + key);
-          bw.addMutation(m);
-        }
+        m = new Mutation(stopRow);
+        ColumnFQ.putDelete(m, Constants.METADATA_CHOPPED_COLUMN);
+        bw.addMutation(m);
         bw.flush();
         
       } catch (Exception ex) {
@@ -1721,6 +1706,36 @@ public class Master implements LiveTServ
         }
       }
     }
+
+    private void deleteTablets(Range scanRange, BatchWriter bw, Connector conn) throws TableNotFoundException,
MutationsRejectedException {
+      Scanner scanner;
+      Mutation m;
+      // Delete everything in the other tablets
+      // group all deletes into tablet into one mutation, this makes tablets
+      // either dissapear entirely or not all.. this is important for the case
+      // where the process terminates in the loop below...
+      scanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+      log.debug("Deleting range " + scanRange);
+      scanner.setRange(scanRange);
+      RowIterator rowIter = new RowIterator(scanner);
+      while (rowIter.hasNext()) {
+        Iterator<Entry<Key,Value>> row = rowIter.next();
+        m = null;
+        while (row.hasNext()) {
+          Entry<Key,Value> entry = row.next();
+          Key key = entry.getKey();
+
+          if (m == null)
+            m = new Mutation(key.getRow());
+          
+          m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
+          log.debug("deleting entry " + key);
+        }
+        bw.addMutation(m);
+      }
+
+      bw.flush();
+    }
     
     private KeyExtent getHighTablet(KeyExtent range) throws AccumuloException {
       try {
@@ -1734,7 +1749,11 @@ public class Master implements LiveTServ
           throw new AccumuloException("No last tablet for a merge " + range);
         }
         Entry<Key,Value> entry = iterator.next();
-        return new KeyExtent(entry.getKey().getRow(), KeyExtent.decodePrevEndRow(entry.getValue()));
+        KeyExtent highTablet = new KeyExtent(entry.getKey().getRow(), KeyExtent.decodePrevEndRow(entry.getValue()));
+        if (highTablet.getTableId() != range.getTableId()) {
+          throw new AccumuloException("No last tablet for merge " + range + " " + highTablet);
+        }
+        return highTablet;
       } catch (Exception ex) {
         throw new AccumuloException("Unexpected failure finding the last tablet for a merge
" + range, ex);
       }

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/master/state/MergeStats.java
Wed Feb 29 21:05:58 2012
@@ -93,7 +93,7 @@ public class MergeStats {
       this.unassigned++;
   }
   
-  public MergeState nextMergeState() throws Exception {
+  public MergeState nextMergeState(Connector connector, CurrentState master) throws Exception
{
     MergeState state = info.getState();
     if (state == MergeState.NONE)
       return state;
@@ -141,7 +141,10 @@ public class MergeStats {
       } else {
         log.info(chopped + " tablets are chopped, " + unassigned + " are offline " + info.getRange());
         if (unassigned == total && chopped == needsToBeChopped) {
-          state = MergeState.MERGING;
+          if (verifyMergeConsistency(connector, master))
+            state = MergeState.MERGING;
+          else
+            log.info("Merge consistency check failed " + info.getRange());
         } else {
           log.info("Waiting for " + unassigned + " unassigned tablets to be " + total + "
" + info.getRange());
         }
@@ -163,7 +166,7 @@ public class MergeStats {
     return state;
   }
   
-  public boolean verifyMergeConsistency(Connector connector, CurrentState master) throws
TableNotFoundException, IOException {
+  private boolean verifyMergeConsistency(Connector connector, CurrentState master) throws
TableNotFoundException, IOException {
     MergeStats verify = new MergeStats(info);
     Scanner scanner = connector.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
     MetaDataTableScanner.configureScanner(scanner, master);

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/BasicServlet.java
Wed Feb 29 21:05:58 2012
@@ -161,6 +161,7 @@ abstract public class BasicServlet exten
     sb.append("<a href='/master'>Master&nbsp;Server</a><br />\n");
     sb.append("<a href='/tservers'>Tablet&nbsp;Servers</a><br />\n");
     sb.append("<a href='/loggers'>Logger&nbsp;Servers</a><br />\n");
+    sb.append("<a href='/vis'>Server Activity</a><br />\n");
     sb.append("<a href='/gc'>Garbage&nbsp;Collector</a><br />\n");
     sb.append("<a href='/tables'>Tables</a><br />\n");
     sb.append("<a href='/trace/summary?minutes=10'>Recent&nbsp;Traces</a><br
/>\n");
@@ -172,7 +173,6 @@ abstract public class BasicServlet exten
     if (numProblems > 0)
       sb.append("<span class='error'><a href='/problems'>Table&nbsp;Problems&nbsp;<span
class='smalltext'>(" + numProblems + ")</a></span></span><br />\n");
     sb.append("<hr />\n");
-    sb.append("<a href='/vis'>Visualization</a><br />\n");
     sb.append("<a href='/xml'>XML</a><hr />\n");
     sb.append("<div class='smalltext'>[<a href='").append("/op?action=refresh&value=").append(refresh
< 1 ? "5" : "-1");
     sb.append("&redir=").append(currentPage(req)).append("'>");

Modified: incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
(original)
+++ incubator/accumulo/trunk/src/server/src/main/java/org/apache/accumulo/server/monitor/servlets/VisServlet.java
Wed Feb 29 21:05:58 2012
@@ -30,13 +30,18 @@ import org.apache.accumulo.server.monito
 public class VisServlet extends BasicServlet {
   private static final long serialVersionUID = 1L;
   boolean useCircles;
-  boolean useIngest;
+  StatType motion;
+  StatType color;
   int spacing;
   String url;
   
+  public enum StatType {
+    osload, ingest, query
+  }
+  
   @Override
   protected String getTitle(HttpServletRequest req) {
-    return "Tablet Server Status Visualization";
+    return "Server Activity";
   }
   
   @Override
@@ -46,15 +51,25 @@ public class VisServlet extends BasicSer
     url = urlsb.toString();
     
     useCircles = true;
-    String shape = req.getParameter("shape");
-    if (shape != null && (shape.equals("square") || shape.equals("squares"))) {
+    String s = req.getParameter("shape");
+    if (s != null && (s.equals("square") || s.equals("squares"))) {
       useCircles = false;
     }
     
-    useIngest = true;
-    String motion = req.getParameter("motion");
-    if (motion != null && (motion.equals("query"))) {
-      useIngest = false;
+    s = req.getParameter("motion");
+    motion = StatType.ingest;
+    if (s != null) {
+      try {
+        motion = StatType.valueOf(s);
+      } catch (Exception e) {}
+    }
+    
+    s = req.getParameter("color");
+    color = StatType.osload;
+    if (s != null) {
+      try {
+        color = StatType.valueOf(s);
+      } catch (Exception e) {}
     }
     
     spacing = 20;
@@ -64,6 +79,8 @@ public class VisServlet extends BasicSer
         spacing = 10;
       else if (size.equals("40"))
         spacing = 40;
+      else if (size.equals("80"))
+        spacing = 80;
     }
     
     ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
@@ -88,12 +105,16 @@ public class VisServlet extends BasicSer
     // size select box
     sb.append("&nbsp;&nbsp<span class='viscontrol'>Size: <select id='size'
onchange='setSize(this)'><option").append(spacing == 10 ? " selected='true'" : "")
         .append(">10</option><option").append(spacing == 20 ? " selected='true'"
: "").append(">20</option><option")
-        .append(spacing == 40 ? " selected='true'" : "").append(">40</option></select></span>\n");
+        .append(spacing == 40 ? " selected='true'" : "").append(">40</option><option").append(spacing
== 80 ? " selected='true'" : "")
+        .append(">80</option></select></span>\n");
     // motion select box
-    sb.append("&nbsp;&nbsp<span class='viscontrol'>Motion: <select id='motion'
onchange='setMotion(this)'><option>Ingest</option><option")
-        .append(!useIngest ? " selected='true'" : "").append(">Query</option></select></span>\n");
+    sb.append("&nbsp;&nbsp<span class='viscontrol'>Motion: <select id='motion'
onchange='setMotion(this)'>");
+    addOptions(sb, motion);
+    sb.append("</select></span>\n");
     // color select box
-    sb.append("&nbsp;&nbsp<span class='viscontrol'>Color: <select><option>OS
Load</option></select></span>\n");
+    sb.append("&nbsp;&nbsp<span class='viscontrol'>Color: <select id='color'
onchange='setColor(this)'>");
+    addOptions(sb, color);
+    sb.append("</select></span>\n");
     sb.append("&nbsp;&nbsp<span class='viscontrol'>(hover for info, click for
details)</span>");
     sb.append("</div>\n\n");
     // floating info box
@@ -110,6 +131,12 @@ public class VisServlet extends BasicSer
     sb.append("</script>\n");
   }
   
+  private void addOptions(StringBuilder sb, StatType selectedStatType) {
+    for (StatType st : StatType.values()) {
+      sb.append("<option").append(st.equals(selectedStatType) ? " selected='true'>"
: ">").append(st).append("</option>");
+    }
+  }
+  
   private void doScript(StringBuilder sb, ArrayList<TabletServerStatus> tservers) {
     InputStream data = VisServlet.class.getClassLoader().getResourceAsStream("web/vis.xml");
     if (data != null) {

Modified: incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml (original)
+++ incubator/accumulo/trunk/src/server/src/main/resources/web/vis.xml Wed Feb 29 21:05:58
2012
@@ -16,12 +16,18 @@
 -->
 
 <script type='text/javascript'>
-var maxLoad = 3*numCores;
-var maxThroughput = 7; // exponent of 10
+// observable stats that can be connected to motion or color
+var statName = ['osload','ingest','query'];
+var speedStatType = 1; // index into statName
+var colorStatType = 0; // index into statName
+var maxStatValue = [numCores, 1000, 10000]; // initial values that are system-dependent will
increase based on observed values
+var significance = [100,1,1]; // values will be converted by floor(this*value)/this
+var maxObservedSpeed = maxStatValue[speedStatType];
+var maxObservedColor = maxStatValue[colorStatType];
 
 // size and spacing variables
 var numDots = 0; // number of dots to draw
-var dotSpacing = 10; // spacing between centers of dots
+var dotSpacing = 10; // spacing between centers of dots (radius)
 var dotPadding = 0.5; // dot padding
 var minDotRadius = 3; // min dot radius
 var maxDotRadius = dotSpacing - dotPadding;
@@ -29,12 +35,11 @@ var maxDotRadius = dotSpacing - dotPaddi
 // arrays of information about each dot
 var dotSize = new Array(numDots); // current sizes
 var dotSizeGrowing = new Array(numDots); // true when dot size is growing, false when shrinking
-var resizeFrameModulus = new Array(numDots); // never resize when <= 0, otherwise resize
when frame % modulus == 0
 var ids = new Array(numDots); // server ids
 var extras = new Array(numDots); // info from which color and motion is derived
 var mousedDot = 0; // the dot currently under the mouse
 
-var maxObservedColor = maxLoad;
+var speed = new Array(numDots); // between 0 (motionless) and maxObservedSpeed (fastest)
 var colors = new Array(numDots); // dot colors between 0 and maxObservedColor, -1 for dead
node
 var colorPalette = ['#0000CC', '#0014B8', '#0029A3', '#003D8F', '#00527A', '#006666', '#007A52',
'#008F3D', '#00A329', '#00B814', '#00CC00', '#14D100', '#29D600', '#3DDB00', '#52E000', '#66E600',
'#7AEB00', '#8FF000', '#A3F500', '#B8FA00', '#CCFF00', '#CCFF00', '#CCF200', '#CCE600', '#CCD900',
'#CCCC00', '#CCBF00', '#CCB200', '#CCA600', '#CC9900', '#CC8C00', '#CC8000', '#CC7300', '#CC6600',
'#CC5900', '#CC4C00', '#CC4000', '#CC3300', '#CC2600', '#CC1A00', '#CC0D00', '#CC0000'];
 
@@ -55,7 +60,6 @@ canvas.addEventListener('click', goToSer
 // initialize settings based on request parameters
 var main = document.getElementById('main');
 var useCircles = true;
-var useIngest = true;
 setShape(document.getElementById('shape'));
 setSize(document.getElementById('size'));
 setMotion(document.getElementById('motion'));
@@ -83,53 +87,55 @@ function handleNewData() {
     xmlReturned = true;
     return;
   }
-  var loadinfo = xmlhttp.responseXML.getElementsByTagName('osload');
-  var ingestinfo = xmlhttp.responseXML.getElementsByTagName('ingest');
-  var queryinfo = xmlhttp.responseXML.getElementsByTagName('query');
+  var statinfo = new Array(statName.length);
+  for (j=0; j < statName.length; j++)
+    statinfo[j] = xmlhttp.responseXML.getElementsByTagName(statName[j]);
   var deadinfo = xmlhttp.responseXML.getElementsByTagName('deadTabletServer');
+  var badinfo = xmlhttp.responseXML.getElementsByTagName('badTabletServer');
   var idinfo = xmlhttp.responseXML.getElementsByTagName('server');
   
-  for (i=0; i < loadinfo.length; i++) {
-    var load = parseFloat(loadinfo[i].childNodes[0].nodeValue);
-    var activity;
-    if (useIngest)
-      activity = parseFloat(ingestinfo[i].childNodes[0].nodeValue);
-    else
-      activity = parseFloat(queryinfo[i].childNodes[0].nodeValue);
+  var statValues = new Array(3);
+  for (i=0; i < idinfo.length; i++) {
     var info = idinfo[i].attributes[0].nodeValue;
-    var extra = '<br>osload: ' + Math.round(load*100)/100 + ', ' + (useIngest?'ingest':'query')
+ ': ' + Math.round(activity);
-    newColor = load;
-    if (activity < 10) {
-      newSkip = -1;
-    } else {
-      newSkip = Math.ceil(Math.pow(1.5,maxThroughput - Math.log(activity)/Math.log(10)));
+    var extra = '<br>';
+    for (j=0; j < statinfo.length; j++) {
+      statValues[j] = Math.floor(significance[j]*parseFloat(statinfo[j][i].childNodes[0].nodeValue))/significance[j];
+      if (maxStatValue[j] < statValues[j])
+        maxStatValue[j] = statValues[j];
+      extra = extra + ' ' + statName[j] + ': ' + statValues[j];
     }
-    setDotInfo(newColor,newSkip,info,extra,i);
+    setDotInfo(statValues[colorStatType],statValues[speedStatType],info,extra,i);
   }
   
-  for (i=loadinfo.length,j=0; j < deadinfo.length; i++,j++) {
+  for (i=idinfo.length,j=0; j < deadinfo.length; i++,j++) {
     setDotInfo(-1,-1,deadinfo[j].attributes[0].nodeValue,'',i);
   }
-  if (numDots != loadinfo.length + deadinfo.length)
+  for (i=idinfo.length+deadinfo.length,j=0; j < badinfo.length; i++,j++) {
+    setDotInfo(-1,-1,badinfo[j].attributes[0].nodeValue,'',i);
+  }
+  if (numDots != idinfo.length + deadinfo.length + badinfo.length)
     drawGrid();
-  numDots = loadinfo.length + deadinfo.length;
+  numDots = idinfo.length + deadinfo.length + badinfo.length;
   xmlReturned = true;
 }
 
-function setDotInfo(color,resizeModulus,id,extra,index) {
+function setDotInfo(color,rate,id,extra,index) {
   if (maxObservedColor < color) {
     maxObservedColor = color;
   }
+  if (maxObservedSpeed < rate) {
+    maxObservedSpeed = rate;
+  }
   if (index >= colors.length) {
     colors.push(color);
-    resizeFrameModulus.push(resizeModulus);
+    speed.push(rate);
     ids.push(id);
     extras.push(extra);
     dotSize.push(maxDotRadius);
     dotSizeGrowing.push(false);
   } else {
     colors[index] = color;
-    resizeFrameModulus[index] = resizeModulus;
+    speed[index] = rate;
     ids[index] = id;
     extras[index] = extra;
     // keep existing size and direction
@@ -151,25 +157,35 @@ function drawDots(){
   var x;
   var y;
   for (i=0; i < numDots; i++) {
-    if (resizeFrameModulus[i]<=0 || dotSize[i] > maxDotRadius) {
+    if (Math.floor(dotSize[i]) > maxDotRadius) {
+      // check for resize by the user
+      dotSize[i] = maxDotRadius;
+    } else if (speed[i]<=0) {
       // if not changing size, increase to max radius
-      // also check for resize by the user
       if (dotSize[i] < maxDotRadius)
         dotSize[i] = dotSize[i] + 1;
-      else
+      if (dotSize[i] > maxDotRadius)
         dotSize[i] = maxDotRadius;
-    } else if (resizeFrameModulus[i] > 0 && frame % resizeFrameModulus[i] == 0)
{
-      if (dotSize[i] >= maxDotRadius) {dotSizeGrowing[i] = false};
-      if (dotSize[i] <= minDotRadius) {dotSizeGrowing[i] = true};
-      if (dotSizeGrowing[i]) {dotSize[i] = dotSize[i] + 1;}
-      else {dotSize[i] = dotSize[i] - 1;}
+    } else {
+      if (dotSizeGrowing[i]) {
+        dotSize[i] = dotSize[i] + speed[i]/maxObservedSpeed;
+        if (dotSize[i] + speed[i]/maxObservedSpeed > maxDotRadius) {
+          dotSizeGrowing[i] = false;
+        }
+      }
+      else {
+        dotSize[i] = dotSize[i] - speed[i]/maxObservedSpeed;
+        if (dotSize[i] - speed[i]/maxObservedSpeed < minDotRadius) {
+          dotSizeGrowing[i] = true;
+        }
+      }
     }
     x = i % width;
     y = Math.floor(i / width);
     if (colors[i]==-1)
       strokeDot(x,y,maxDotRadius-1,deadColor);
     else
-      drawDot(x,y,dotSize[i],getColor(colors[i]));
+      drawDot(x,y,Math.floor(dotSize[i]),getColor(colors[i]));
   }
   if (mousedDot < numDots)
     document.getElementById('vishoverinfo').innerHTML=ids[mousedDot]+extras[mousedDot];
@@ -178,9 +194,9 @@ function drawDots(){
 // fill in a few grey dots
 function drawGrid() {
   context.clearRect(0, 0, canvas.width, canvas.height);
-  for (i=0; i < 32; i++) {
-    for (j=0; j < 32; j++) {
-      drawDot(i,j,maxDotRadius,nullColor);
+  for (i=0,k=0; i < canvas.width; i+=dotSpacing*2,k++) {
+    for (j=0,l=0; j < canvas.height; j+=dotSpacing*2,l++) {
+      drawDot(k,l,maxDotRadius,nullColor);
     }
   }
 }
@@ -271,6 +287,10 @@ function setSize(obj) {
       dotSpacing = 20;
       minDotRadius = 5;
       break;
+    case 3:
+      dotSpacing = 40;
+      minDotRadius = 7;
+      break;
     default:
       dotSpacing = 10;
       minDotRadius = 3;
@@ -282,16 +302,16 @@ function setSize(obj) {
 
 // callback for motion selection
 function setMotion(obj) {
-  switch (obj.selectedIndex) {
-    case 0:
-      useIngest = true;
-      break;
-    case 1:
-      useIngest = false;
-      break;
-    default:
-      useIngest = true;
-  }
+  speedStatType = obj.selectedIndex;
+  maxObservedSpeed = maxStatValue[speedStatType];
+  drawGrid();
+  setState();
+}
+
+// callback for color selection
+function setColor(obj) {
+  colorStatType = obj.selectedIndex;
+  maxObservedColor = maxStatValue[colorStatType];
   drawGrid();
   setState();
 }
@@ -328,8 +348,8 @@ function showId(e) {
 }
 
 function setState() {
-  var url = visurl+'?shape='+(useCircles?'circles':'squares')+'&size='+(dotSpacing*2)+'&motion='+(useIngest?'ingest':'query');
-  window.history.replaceState(window.history.state,'Tablet Server Status Visualization',url);
+  var url = visurl+'?shape='+(useCircles?'circles':'squares')+'&size='+(dotSpacing*2)+'&motion='+statName[speedStatType]+'&color='+statName[colorStatType];
+  window.history.replaceState(window.history.state,'Server Activity',url);
 }
 
 // go to server page on click

Modified: incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java?rev=1295272&r1=1295271&r2=1295272&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
(original)
+++ incubator/accumulo/trunk/src/server/src/test/java/org/apache/accumulo/server/master/TestMergeState.java
Wed Feb 29 21:05:58 2012
@@ -132,7 +132,7 @@ public class TestMergeState {
     
     // do the state check
     MergeStats stats = scan(state, metaDataStateStore);
-    MergeState newState = stats.nextMergeState();
+    MergeState newState = stats.nextMergeState(connector, state);
     Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
     
     // unassign the tablets
@@ -141,12 +141,9 @@ public class TestMergeState {
     deleter.setRanges(Collections.singletonList(new Range()));
     deleter.delete();
     
-    // now we should be ready to merge
+    // now we should be ready to merge but, we have an inconsistent !METADATA table
     stats = scan(state, metaDataStateStore);
-    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState());
-
-    // but, we have an inconsistent !METADATA table, so double check
-    Assert.assertFalse(stats.verifyMergeConsistency(connector, state));
+    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
     
     // finish the split
     KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
@@ -157,7 +154,7 @@ public class TestMergeState {
     
     // onos... there's a new tablet online
     stats = scan(state, metaDataStateStore);
-    Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState());
+    Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));
     
     // chop it
     m = tablet.getPrevRowUpdateMutation();
@@ -165,7 +162,7 @@ public class TestMergeState {
     update(connector, m);
 
     stats = scan(state, metaDataStateStore);
-    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState());
+    Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
 
     // take it offline
     m = tablet.getPrevRowUpdateMutation();
@@ -174,10 +171,7 @@ public class TestMergeState {
     
     // now we can split
     stats = scan(state, metaDataStateStore);
-    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState());
-
-    // and we have consistent !METADATA table
-    Assert.assertTrue(stats.verifyMergeConsistency(connector, state));
+    Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));
 
   }
 



Mime
View raw message