hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bra...@apache.org
Subject [28/50] [abbrv] hadoop git commit: HDDS-754. VolumeInfo#getScmUsed throws NPE. Contributed by Hanisha Koneru.
Date Fri, 02 Nov 2018 09:29:36 GMT
HDDS-754. VolumeInfo#getScmUsed throws NPE.
Contributed by Hanisha Koneru.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773f0d15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773f0d15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773f0d15

Branch: refs/heads/HDFS-13891
Commit: 773f0d1519715e3ddf77c139998cc12d7447da66
Parents: e33b61f
Author: Anu Engineer <aengineer@apache.org>
Authored: Tue Oct 30 19:17:57 2018 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Tue Oct 30 19:17:57 2018 -0700

----------------------------------------------------------------------
 .../container/common/volume/VolumeInfo.java      | 19 +++++++++++++++++--
 .../ozone/container/common/volume/VolumeSet.java | 11 +++++++----
 .../container/common/volume/TestHddsVolume.java  |  9 ++++++---
 .../container/common/volume/TestVolumeSet.java   |  4 +++-
 .../hdds/scm/pipeline/TestNodeFailure.java       |  3 ++-
 .../apache/hadoop/ozone/MiniOzoneCluster.java    |  8 ++++++++
 .../hadoop/ozone/MiniOzoneClusterImpl.java       | 16 +++++++++++++++-
 7 files changed, 58 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
index 62fca63..0de9f18 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfo.java
@@ -95,15 +95,30 @@ public class VolumeInfo {
     this.usage = new VolumeUsage(root, b.conf);
   }
 
-  public long getCapacity() {
-    return configuredCapacity < 0 ? usage.getCapacity() : configuredCapacity;
+  public long getCapacity() throws IOException {
+    if (configuredCapacity < 0) {
+      if (usage == null) {
+        throw new IOException("Volume Usage thread is not running. This error" +
+            " is usually seen during DataNode shutdown.");
+      }
+      return usage.getCapacity();
+    }
+    return configuredCapacity;
   }
 
   public long getAvailable() throws IOException {
+    if (usage == null) {
+      throw new IOException("Volume Usage thread is not running. This error " +
+          "is usually seen during DataNode shutdown.");
+    }
     return usage.getAvailable();
   }
 
   public long getScmUsed() throws IOException {
+    if (usage == null) {
+      throw new IOException("Volume Usage thread is not running. This error " +
+          "is usually seen during DataNode shutdown.");
+    }
     return usage.getScmUsed();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
index 5b6b823..d30dd89 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeSet.java
@@ -372,18 +372,21 @@ public class VolumeSet {
       for (Map.Entry<String, HddsVolume> entry : volumeMap.entrySet()) {
         hddsVolume = entry.getValue();
         VolumeInfo volumeInfo = hddsVolume.getVolumeInfo();
-        long scmUsed = 0;
-        long remaining = 0;
+        long scmUsed;
+        long remaining;
+        long capacity;
         failed = false;
         try {
           scmUsed = volumeInfo.getScmUsed();
           remaining = volumeInfo.getAvailable();
+          capacity = volumeInfo.getCapacity();
         } catch (IOException ex) {
           LOG.warn("Failed to get scmUsed and remaining for container " +
-              "storage location {}", volumeInfo.getRootDir());
+              "storage location {}", volumeInfo.getRootDir(), ex);
           // reset scmUsed and remaining if df/du failed.
           scmUsed = 0;
           remaining = 0;
+          capacity = 0;
           failed = true;
         }
 
@@ -392,7 +395,7 @@ public class VolumeSet {
         builder.setStorageLocation(volumeInfo.getRootDir())
             .setId(hddsVolume.getStorageID())
             .setFailed(failed)
-            .setCapacity(hddsVolume.getCapacity())
+            .setCapacity(capacity)
             .setRemaining(remaining)
             .setScmUsed(scmUsed)
             .setStorageType(hddsVolume.getStorageType());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
index 7755345..6b46762 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java
@@ -31,6 +31,7 @@ import org.junit.rules.TemporaryFolder;
 import org.mockito.Mockito;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Properties;
 import java.util.UUID;
 
@@ -134,12 +135,14 @@ public class TestHddsVolume {
         scmUsedFile.exists());
 
     try {
-      // Volume.getAvailable() should fail with NullPointerException as usage
-      // is shutdown.
+      // Volume.getAvailable() should fail with IOException
+      // as usage thread is shutdown.
       volume.getAvailable();
       fail("HddsVolume#shutdown test failed");
     } catch (Exception ex){
-      assertTrue(ex instanceof NullPointerException);
+      assertTrue(ex instanceof IOException);
+      assertTrue(ex.getMessage().contains(
+          "Volume Usage thread is not running."));
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
index fca68b1..7bb8a43 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java
@@ -222,8 +222,10 @@ public class TestVolumeSet {
         // getAvailable() should throw null pointer exception as usage is null.
         volume.getAvailable();
         fail("Volume shutdown failed.");
-      } catch (NullPointerException ex) {
+      } catch (IOException ex) {
         // Do Nothing. Exception is expected.
+        assertTrue(ex.getMessage().contains(
+            "Volume Usage thread is not running."));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
index 9a1c705..618cd8e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java
@@ -99,7 +99,7 @@ public class TestNodeFailure {
     }
   }
 
-  @Test
+  @Test(timeout = 300_000L)
   public void testPipelineFail() throws InterruptedException, IOException,
       TimeoutException {
     Assert.assertEquals(ratisContainer1.getPipeline().getPipelineState(),
@@ -118,6 +118,7 @@ public class TestNodeFailure {
         pipelineManager.getPipeline(ratisContainer2.getPipeline().getId())
             .getPipelineState());
     // Now restart the datanode and make sure that a new pipeline is created.
+    cluster.setWaitForClusterToBeReadyTimeout(300000);
     cluster.restartHddsDatanode(dnToFail, true);
     ContainerWithPipeline ratisContainer3 =
         containerManager.allocateContainer(RATIS, THREE, "testOwner");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3aad7f7..15bf8d0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -67,6 +67,14 @@ public interface MiniOzoneCluster {
   void waitForClusterToBeReady() throws TimeoutException, InterruptedException;
 
   /**
+   * Sets the timeout value after which
+   * {@link MiniOzoneCluster#waitForClusterToBeReady} times out.
+   *
+   * @param timeoutInMs timeout value in milliseconds
+   */
+  void setWaitForClusterToBeReadyTimeout(int timeoutInMs);
+
+  /**
    * Waits/blocks till the cluster is out of chill mode.
    *
    * @throws TimeoutException TimeoutException In case of timeout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/773f0d15/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 11bc0e0..6c0f408 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -90,6 +90,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
   private final OzoneManager ozoneManager;
   private final List<HddsDatanodeService> hddsDatanodes;
 
+  // Timeout for the cluster to be ready
+  private int waitForClusterToBeReadyTimeout = 60000; // 1 min
+
   /**
    * Creates a new MiniOzoneCluster.
    *
@@ -122,7 +125,18 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
           isReady? "Cluster is ready" : "Waiting for cluster to be ready",
           healthy, hddsDatanodes.size());
       return isReady;
-    }, 1000, 60 * 1000); //wait for 1 min.
+    }, 1000, waitForClusterToBeReadyTimeout);
+  }
+
+  /**
+   * Sets the timeout value after which
+   * {@link MiniOzoneClusterImpl#waitForClusterToBeReady} times out.
+   *
+   * @param timeoutInMs timeout value in milliseconds
+   */
+  @Override
+  public void setWaitForClusterToBeReadyTimeout(int timeoutInMs) {
+    waitForClusterToBeReadyTimeout = timeoutInMs;
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message