hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HDFS-11631. Block Storage : allow cblock server to be started from hdfs commandline. Contributed by Chen Liang.
Date Mon, 17 Apr 2017 23:01:41 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 49c92ea73 -> 350220bfb


HDFS-11631. Block Storage : allow cblock server to be started from hdfs commandline. Contributed
by Chen Liang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/350220bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/350220bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/350220bf

Branch: refs/heads/HDFS-7240
Commit: 350220bfb3c555ddf8ab4f0f5a444e37a6d3d9db
Parents: 49c92ea
Author: Anu Engineer <aengineer@apache.org>
Authored: Mon Apr 17 15:54:39 2017 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Mon Apr 17 15:54:39 2017 -0700

----------------------------------------------------------------------
 .../hadoop-hdfs/src/main/bin/hdfs               | 11 +++++
 .../apache/hadoop/cblock/CBlockConfigKeys.java  | 15 ++++++-
 .../org/apache/hadoop/cblock/CBlockManager.java | 47 ++++++++++++++++++++
 3 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 1ae5de4..afb27c3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -33,6 +33,8 @@ function hadoop_usage
 
   hadoop_add_subcommand "balancer" "run a cluster balancing utility"
   hadoop_add_subcommand "cacheadmin" "configure the HDFS cache"
+  hadoop_add_subcommand "cblock" "cblock CLI"
+  hadoop_add_subcommand "cblockserver" "run cblock server"
   hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and
the required libraries"
   hadoop_add_subcommand "crypto" "configure HDFS encryption zones"
   hadoop_add_subcommand "datanode" "run a DFS datanode"
@@ -86,6 +88,15 @@ function hdfscmd_case
     cacheadmin)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
     ;;
+    cblock)
+      HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli
+    ;;
+    cblockserver)
+      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+      HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager
+      hadoop_debug "Appending HADOOP_CBLOCK_OPTS onto HADOOP_OPTS"
+      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CBLOCK_OPTS}"
+    ;;
     classpath)
       hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
     ;;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
index 8aa4ae8..b1fba41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
@@ -142,9 +142,20 @@ public final class CBlockConfigKeys {
   public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
       "127.0.0.1";
 
+  // to what address cblock server should talk to scm?
+  public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
+      "dfs.cblock.scm.ipaddress";
+  public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
+      "127.0.0.1";
+  public static final String DFS_CBLOCK_SCM_PORT_KEY =
+      "dfs.cblock.scm.port";
+  public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
+
   public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
-      "dfs.cblock.container.size.gb";
-  public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = 5;
+      "dfs.cblock.container.size";
+  public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
+      5;
+
 
   private CBlockConfigKeys() {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/350220bf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
index c7abed6..9f8d5b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
@@ -30,6 +30,11 @@ import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
 import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolServerSideTranslatorPB;
 import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
 import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.client.ContainerOperationClient;
 import org.apache.hadoop.scm.client.ScmClient;
 import org.apache.hadoop.cblock.storage.StorageManager;
 import org.apache.hadoop.cblock.util.KeyUtil;
@@ -37,6 +42,9 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.utils.LevelDBStore;
 import org.iq80.leveldb.DBIterator;
 import org.slf4j.Logger;
@@ -50,9 +58,15 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_KEY;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
@@ -60,6 +74,7 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HA
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
 import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT;
 
 /**
  * The main entry point of CBlock operations, ALL the CBlock operations
@@ -317,4 +332,36 @@ public class CBlockManager implements CBlockServiceProtocol,
     }
     return response;
   }
+
+  public static void main(String[] args) throws Exception {
+    long version = RPC.getProtocolVersion(
+        StorageContainerLocationProtocolPB.class);
+    OzoneConfiguration ozoneConf = new OzoneConfiguration();
+    String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
+        DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
+    int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
+        DFS_CBLOCK_SCM_PORT_DEFAULT);
+    int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
+        DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
+    ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
+    InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
+
+    ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
+        OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
+    LOG.info(
+        "Creating StorageContainerLocationProtocol RPC client with address {}",
+        address);
+    RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
+        ProtobufRpcEngine.class);
+    StorageContainerLocationProtocolClientSideTranslatorPB client =
+        new StorageContainerLocationProtocolClientSideTranslatorPB(
+            RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
+                address, UserGroupInformation.getCurrentUser(), ozoneConf,
+                NetUtils.getDefaultSocketFactory(ozoneConf),
+                Client.getRpcTimeout(ozoneConf)));
+    ScmClient storageClient = new ContainerOperationClient(
+        client, new XceiverClientManager(ozoneConf));
+    CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
+    cbm.start();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message