accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject [accumulo] branch 2.0 updated: Improve error handling in Monitor. Fixes #1258 (#1262)
Date Wed, 10 Jul 2019 20:09:26 GMT
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.0 by this push:
     new 477a9cd  Improve error handling in Monitor. Fixes #1258 (#1262)
477a9cd is described below

commit 477a9cd275243763e9f1029cb91b4707be714450
Author: Mike Miller <mmiller@apache.org>
AuthorDate: Wed Jul 10 16:09:22 2019 -0400

    Improve error handling in Monitor. Fixes #1258 (#1262)
    
    * Make rest calls fail fast and return empty objects
    * This will make the Monitor more stable when Master is down
---
 .../java/org/apache/accumulo/monitor/Monitor.java  |  1 +
 .../rest/bulkImports/BulkImportResource.java       |  9 +++--
 .../monitor/rest/master/MasterResource.java        |  6 +++-
 .../accumulo/monitor/rest/scans/ScansResource.java |  8 +++--
 .../monitor/rest/status/StatusResource.java        | 10 +++---
 .../monitor/rest/tables/TablesResource.java        | 38 ++++++++++++----------
 .../rest/tservers/TabletServerResource.java        | 13 ++++----
 7 files changed, 52 insertions(+), 33 deletions(-)

diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 929d1c0..4785671 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -256,6 +256,7 @@ public class Monitor extends AbstractServer implements HighlyAvailableService
{
             retry = false;
           } else {
             mmi = null;
+            log.error("Unable to get info from Master");
           }
           gcStatus = fetchGcStatus();
         } catch (Exception e) {
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
index d337e53..3b4a3a6 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/bulkImports/BulkImportResource.java
@@ -25,6 +25,7 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.accumulo.core.master.thrift.BulkImportStatus;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.monitor.Monitor;
 
@@ -48,17 +49,19 @@ public class BulkImportResource {
    */
   @GET
   public BulkImport getTables() {
-
     BulkImport bulkImport = new BulkImport();
+    MasterMonitorInfo mmi = monitor.getMmi();
+    if (mmi == null)
+      return bulkImport;
 
     // Generating Bulk Import and adding it to the return object
-    for (BulkImportStatus bulk : monitor.getMmi().bulkImports) {
+    for (BulkImportStatus bulk : mmi.bulkImports) {
       bulkImport
           .addBulkImport(new BulkImportInformation(bulk.filename, bulk.startTime, bulk.state));
     }
 
     // Generating TServer Bulk Import and adding it to the return object
-    for (TabletServerStatus tserverInfo : monitor.getMmi().getTServerInfo()) {
+    for (TabletServerStatus tserverInfo : mmi.getTServerInfo()) {
       int size = 0;
       long oldest = 0L;
 
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
index d29912e..e937868 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/master/MasterResource.java
@@ -232,9 +232,13 @@ public class MasterResource {
    * @return servers shutting down list
    */
   public static ServersShuttingDown getServersShuttingDown(Monitor monitor) {
+    MasterMonitorInfo mmi = monitor.getMmi();
     ServersShuttingDown servers = new ServersShuttingDown();
+    if (mmi == null)
+      return servers;
+
     // Add new servers to the list
-    for (String server : monitor.getMmi().serversShuttingDown) {
+    for (String server : mmi.serversShuttingDown) {
       servers.addServerShuttingDown(new ServerShuttingDownInformation(server));
     }
     return servers;
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
index 7b211d5..738f90f 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
@@ -24,6 +24,7 @@ import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.util.HostAndPort;
 import org.apache.accumulo.monitor.Monitor;
@@ -48,13 +49,16 @@ public class ScansResource {
    */
   @GET
   public Scans getTables() {
-
     Scans scans = new Scans();
+    MasterMonitorInfo mmi = monitor.getMmi();
+    if (mmi == null) {
+      return scans;
+    }
 
     Map<HostAndPort,ScanStats> entry = monitor.getScans();
 
     // Adds new scans to the array
-    for (TabletServerStatus tserverInfo : monitor.getMmi().getTServerInfo()) {
+    for (TabletServerStatus tserverInfo : mmi.getTServerInfo()) {
       ScanStats stats = entry.get(HostAndPort.fromString(tserverInfo.name));
       if (stats != null) {
         scans.addScan(new ScanInformation(tserverInfo, stats.scanCount, stats.oldestScan));
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
index c89b370..8ac76e9 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/status/StatusResource.java
@@ -24,6 +24,7 @@ import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.MediaType;
 
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.monitor.Monitor;
 import org.apache.accumulo.server.monitor.DedupedLogEvent;
 import org.apache.accumulo.server.monitor.LogService;
@@ -57,8 +58,9 @@ public class StatusResource {
     Status masterStatus;
     Status gcStatus;
     Status tServerStatus = Status.ERROR;
+    MasterMonitorInfo mmi = monitor.getMmi();
 
-    if (monitor.getMmi() != null) {
+    if (mmi != null) {
       if (monitor.getGcStatus() != null) {
         gcStatus = Status.OK;
       } else {
@@ -68,9 +70,9 @@ public class StatusResource {
       List<String> masters = monitor.getContext().getMasterLocations();
       masterStatus = masters.size() == 0 ? Status.ERROR : Status.OK;
 
-      int tServerUp = monitor.getMmi().getTServerInfoSize();
-      int tServerDown = monitor.getMmi().getDeadTabletServersSize();
-      int tServerBad = monitor.getMmi().getBadTServersSize();
+      int tServerUp = mmi.getTServerInfoSize();
+      int tServerDown = mmi.getDeadTabletServersSize();
+      int tServerBad = mmi.getBadTServersSize();
 
       /*
        * If there are no dead or bad servers and there are tservers up, status is OK, if
there are
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
index ccabfc4..949ea32 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java
@@ -38,6 +38,7 @@ import org.apache.accumulo.core.clientImpl.Tables;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.metadata.MetadataTable;
@@ -79,15 +80,19 @@ public class TablesResource {
 
   public static TableInformationList getTables(Monitor monitor) {
     TableInformationList tableList = new TableInformationList();
+    MasterMonitorInfo mmi = monitor.getMmi();
+    if (mmi == null) {
+      return tableList;
+    }
     SortedMap<TableId,TableInfo> tableStats = new TreeMap<>();
 
-    if (monitor.getMmi() != null && monitor.getMmi().tableMap != null) {
-      for (Map.Entry<String,TableInfo> te : monitor.getMmi().tableMap.entrySet()) {
+    if (mmi.tableMap != null) {
+      for (Map.Entry<String,TableInfo> te : mmi.tableMap.entrySet()) {
         tableStats.put(TableId.of(te.getKey()), te.getValue());
       }
     }
 
-    Map<String,Double> compactingByTable = TableInfoUtil.summarizeTableStats(monitor.getMmi());
+    Map<String,Double> compactingByTable = TableInfoUtil.summarizeTableStats(mmi);
     TableManager tableManager = monitor.getContext().getTableManager();
 
     // Add tables to the list
@@ -125,8 +130,13 @@ public class TablesResource {
       regexp = ALPHA_NUM_REGEX_TABLE_ID) String tableIdStr) {
     String rootTabletLocation = monitor.getContext().getRootTabletLocation();
     TableId tableId = TableId.of(tableIdStr);
+    MasterMonitorInfo mmi = monitor.getMmi();
+    // fail fast if unable to get monitor info
+    if (mmi == null) {
+      return new TabletServers();
+    }
 
-    TabletServers tabletServers = new TabletServers(monitor.getMmi().tServerInfo.size());
+    TabletServers tabletServers = new TabletServers(mmi.tServerInfo.size());
 
     if (StringUtils.isBlank(tableIdStr)) {
       return tabletServers;
@@ -158,15 +168,13 @@ public class TablesResource {
     }
 
     List<TabletServerStatus> tservers = new ArrayList<>();
-    if (monitor.getMmi() != null) {
-      for (TabletServerStatus tss : monitor.getMmi().tServerInfo) {
-        try {
-          if (tss.name != null && locs.contains(tss.name)) {
-            tservers.add(tss);
-          }
-        } catch (Exception ex) {
-          return tabletServers;
+    for (TabletServerStatus tss : mmi.tServerInfo) {
+      try {
+        if (tss.name != null && locs.contains(tss.name)) {
+          tservers.add(tss);
         }
+      } catch (Exception ex) {
+        return tabletServers;
       }
     }
 
@@ -175,10 +183,7 @@ public class TablesResource {
       if (status == null) {
         status = NO_STATUS;
       }
-      TableInfo summary = TableInfoUtil.summarizeTableStats(status);
-      if (tableId != null) {
-        summary = status.tableMap.get(tableId.canonical());
-      }
+      TableInfo summary = status.tableMap.get(tableId.canonical());
       if (summary == null) {
         continue;
       }
@@ -190,7 +195,6 @@ public class TablesResource {
     }
 
     return tabletServers;
-
   }
 
 }
diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
index df80133..e3ef081 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tservers/TabletServerResource.java
@@ -34,9 +34,7 @@ import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response.Status;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.clientImpl.ClientContext;
@@ -83,7 +81,7 @@ public class TabletServerResource {
   public TabletServers getTserverSummary() {
     MasterMonitorInfo mmi = monitor.getMmi();
     if (mmi == null) {
-      throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
+      return new TabletServers();
     }
 
     TabletServers tserverInfo = new TabletServers(mmi.tServerInfo.size());
@@ -123,7 +121,7 @@ public class TabletServerResource {
 
     MasterMonitorInfo mmi = monitor.getMmi();
     if (mmi == null) {
-      throw new WebApplicationException(Status.INTERNAL_SERVER_ERROR);
+      return new TabletServersRecovery();
     }
 
     for (TabletServerStatus server : mmi.tServerInfo) {
@@ -154,9 +152,12 @@ public class TabletServerResource {
   public TabletServerSummary getTserverDetails(
       @PathParam("address") @NotNull @Pattern(regexp = HOSTNAME_PORT_REGEX) String tserverAddress)
       throws Exception {
+    MasterMonitorInfo mmi = monitor.getMmi();
+    if (mmi == null)
+      return new TabletServerSummary();
 
     boolean tserverExists = false;
-    for (TabletServerStatus ts : monitor.getMmi().getTServerInfo()) {
+    for (TabletServerStatus ts : mmi.getTServerInfo()) {
       if (tserverAddress.equals(ts.getName())) {
         tserverExists = true;
         break;
@@ -188,7 +189,7 @@ public class TabletServerResource {
       TabletClientService.Client client =
           ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context);
       try {
-        for (String tableId : monitor.getMmi().tableMap.keySet()) {
+        for (String tableId : mmi.tableMap.keySet()) {
           tsStats.addAll(client.getTabletStats(TraceUtil.traceInfo(), context.rpcCreds(),
tableId));
         }
         historical = client.getHistoricalStats(TraceUtil.traceInfo(), context.rpcCreds());


Mime
View raw message