hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDDS-624.PutBlock fails with Unexpected Storage Container Exception. Contributed by Arpit Agarwal.
Date Fri, 12 Oct 2018 21:30:14 GMT
Repository: hadoop
Updated Branches:
  refs/heads/ozone-0.3 252a44f8c -> 7daeaa9a9


HDDS-624.PutBlock fails with Unexpected Storage Container Exception.
Contributed by  Arpit Agarwal.

(cherry picked from commit 02e1ef5e0779369f1363df2bb0437945fe9a271c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7daeaa9a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7daeaa9a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7daeaa9a

Branch: refs/heads/ozone-0.3
Commit: 7daeaa9a983b3ded49a4e151e69210123bc9b273
Parents: 252a44f
Author: Anu Engineer <aengineer@apache.org>
Authored: Fri Oct 12 14:22:46 2018 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Fri Oct 12 14:29:59 2018 -0700

----------------------------------------------------------------------
 .../hadoop/utils/MetadataStoreBuilder.java      | 38 ++++++++++----------
 .../hadoop/utils/TestRocksDBStoreMBean.java     | 19 ++++++----
 .../container/common/utils/ContainerCache.java  |  6 ++--
 .../container/keyvalue/helpers/BlockUtils.java  |  3 +-
 4 files changed, 38 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7daeaa9a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
index 85cebed..a1c9b85 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.utils;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.iq80.leveldb.Options;
 import org.rocksdb.BlockBasedTableConfig;
@@ -30,6 +31,8 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Optional;
+import java.util.function.Supplier;
 
 import static org.apache.hadoop.ozone.OzoneConfigKeys
     .OZONE_METADATA_STORE_IMPL_LEVELDB;
@@ -53,7 +56,7 @@ public class MetadataStoreBuilder {
   private File dbFile;
   private long cacheSize;
   private boolean createIfMissing = true;
-  private Configuration conf;
+  private Optional<Configuration> optionalConf = Optional.empty();
   private String dbType;
 
   public static MetadataStoreBuilder newBuilder() {
@@ -76,7 +79,7 @@ public class MetadataStoreBuilder {
   }
 
   public MetadataStoreBuilder setConf(Configuration configuration) {
-    this.conf = configuration;
+    this.optionalConf = Optional.of(configuration);
     return this;
   }
 
@@ -98,13 +101,12 @@ public class MetadataStoreBuilder {
     }
 
     // Build db store based on configuration
-    MetadataStore store = null;
+    final Configuration conf = optionalConf.orElseGet(
+        () -> new OzoneConfiguration());
 
     if(dbType == null) {
       LOG.debug("dbType is null, using ");
-      dbType = conf == null ?
-          OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT :
-          conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+      dbType = conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
               OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
       LOG.debug("dbType is null, using dbType {} from ozone configuration",
           dbType);
@@ -117,7 +119,7 @@ public class MetadataStoreBuilder {
       if (cacheSize > 0) {
         options.cacheSize(cacheSize);
       }
-      store = new LevelDBStore(dbFile, options);
+      return new LevelDBStore(dbFile, options);
     } else if (OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(dbType)) {
       org.rocksdb.Options opts = new org.rocksdb.Options();
       opts.setCreateIfMissing(createIfMissing);
@@ -128,10 +130,9 @@ public class MetadataStoreBuilder {
         opts.setTableFormatConfig(tableConfig);
       }
 
-      String rocksDbStat = conf == null ?
-          OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT :
-          conf.getTrimmed(OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
-              OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
+      String rocksDbStat = conf.getTrimmed(
+          OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
+          OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
 
       if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
         Statistics statistics = new Statistics();
@@ -139,14 +140,13 @@ public class MetadataStoreBuilder {
         opts = opts.setStatistics(statistics);
 
       }
-      store = new RocksDBStore(dbFile, opts);
-    } else {
-      throw new IllegalArgumentException("Invalid argument for "
-          + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
-          + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
-          + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
-          + ", but met " + dbType);
+      return new RocksDBStore(dbFile, opts);
     }
-    return store;
+    
+    throw new IllegalArgumentException("Invalid argument for "
+        + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
+        + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
+        + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
+        + ", but met " + dbType);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7daeaa9a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
index a7ce60b..db2572c 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestRocksDBStoreMBean.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 import javax.management.MBeanServer;
@@ -32,15 +33,24 @@ import java.lang.management.ManagementFactory;
  * Test the JMX interface for the rocksdb metastore implementation.
  */
 public class TestRocksDBStoreMBean {
+  
+  Configuration conf;
+  
+  @Before
+  public void init() throws Exception {
+    conf = new OzoneConfiguration();
+
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+        OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
+  }
+  
 
   @Test
   public void testJmxBeans() throws Exception {
     File testDir =
         GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
 
-    Configuration conf = new OzoneConfiguration();
-    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
-        OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
+    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL");
 
     RocksDBStore metadataStore =
         (RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
@@ -72,9 +82,6 @@ public class TestRocksDBStoreMBean {
     File testDir = GenericTestUtils
         .getTestDir(getClass().getSimpleName() + "-withoutstat");
 
-    Configuration conf = new OzoneConfiguration();
-    conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
-        OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
     conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
         OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7daeaa9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
index c63eb73..a533684 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
@@ -123,10 +123,11 @@ public final class ContainerCache extends LRUMap {
    * @param containerID - ID of the container.
    * @param containerDBType - DB type of the container.
    * @param containerDBPath - DB path of the container.
+   * @param conf - Hadoop Configuration.
    * @return MetadataStore.
    */
-  public MetadataStore getDB(long containerID, String containerDBType, String
-                             containerDBPath)
+  public MetadataStore getDB(long containerID, String containerDBType,
+                             String containerDBPath, Configuration conf)
       throws IOException {
     Preconditions.checkState(containerID >= 0,
         "Container ID cannot be negative.");
@@ -138,6 +139,7 @@ public final class ContainerCache extends LRUMap {
         db = MetadataStoreBuilder.newBuilder()
             .setDbFile(new File(containerDBPath))
             .setCreateIfMissing(false)
+            .setConf(conf)
             .setDBType(containerDBType)
             .build();
         this.put(containerID, db);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7daeaa9a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
index b25fec4..d6cadc8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java
@@ -75,7 +75,8 @@ public final class BlockUtils {
     Preconditions.checkNotNull(containerData.getDbFile());
     try {
       return cache.getDB(containerData.getContainerID(), containerData
-          .getContainerDBType(), containerData.getDbFile().getAbsolutePath());
+          .getContainerDBType(), containerData.getDbFile().getAbsolutePath(),
+          conf);
     } catch (IOException ex) {
       String message = String.format("Error opening DB. Container:%s " +
           "ContainerPath:%s", containerData.getContainerID(), containerData


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message