hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject [14/45] hadoop git commit: HDDS-94. Change ozone datanode command to start the standalone datanode plugin. Contributed by Sandeep Nemuri.
Date Mon, 02 Jul 2018 20:32:31 GMT
HDDS-94. Change ozone datanode command to start the standalone datanode plugin.
Contributed by  Sandeep Nemuri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18932717
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18932717
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18932717

Branch: refs/heads/HDDS-4
Commit: 18932717c42382ed8842de7719ec6d20c1765366
Parents: 1e30547
Author: Anu Engineer <aengineer@apache.org>
Authored: Tue Jun 26 18:28:47 2018 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Wed Jun 27 14:15:15 2018 -0700

----------------------------------------------------------------------
 hadoop-dist/src/main/compose/ozone/docker-compose.yaml | 12 ------------
 hadoop-dist/src/main/compose/ozone/docker-config       |  5 -----
 .../src/main/compose/ozoneperf/docker-compose.yaml     | 13 -------------
 hadoop-dist/src/main/compose/ozoneperf/docker-config   |  5 -----
 .../org/apache/hadoop/ozone/HddsDatanodeService.java   | 13 ++++++++++++-
 .../src/test/acceptance/basic/docker-compose.yaml      | 12 ------------
 .../src/test/acceptance/basic/docker-config            |  5 -----
 .../src/test/acceptance/commonlib.robot                |  1 -
 .../src/test/acceptance/ozonefs/docker-compose.yaml    | 12 ------------
 .../src/test/acceptance/ozonefs/docker-config          |  5 -----
 hadoop-ozone/common/src/main/bin/ozone                 |  9 +++------
 11 files changed, 15 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
index faf420c..512c649 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ../../ozone:/opt/hadoop
-      ports:
-         - 9870:9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
index c693db0..632f870 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
index fb7873b..3233c11 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
@@ -16,19 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ../../ozone:/opt/hadoop
-         - ./jmxpromo.jar:/opt/jmxpromo.jar
-      ports:
-         - 9870:9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config
index e4f5485..2be22a7 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index fa4187a..ddeec87 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,9 +25,11 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.statemachine
     .DatanodeStateMachine;
+import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ServicePlugin;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
@@ -229,9 +231,18 @@ public class HddsDatanodeService implements ServicePlugin {
 
   public static void main(String[] args) {
     try {
+      if (DFSUtil.parseHelpArgument(args, "Starts HDDS Datanode", System.out, false)) {
+        System.exit(0);
+      }
+      Configuration conf = new OzoneConfiguration();
+      GenericOptionsParser hParser = new GenericOptionsParser(conf, args);
+      if (!hParser.isParseSuccessful()) {
+        GenericOptionsParser.printGenericCommandUsage(System.err);
+        System.exit(1);
+      }
       StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG);
       HddsDatanodeService hddsDatanodeService =
-          createHddsDatanodeService(new OzoneConfiguration());
+          createHddsDatanodeService(conf);
       hddsDatanodeService.start(null);
       hddsDatanodeService.join();
     } catch (Throwable e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
index 44bd4a0..b50f42d 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
index 180dc8e..c3ec2ef 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
 OZONE-SITE.XML_ozone.scm.names=scm
@@ -24,13 +23,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
 OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
index 01ed302..a5ea30a 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
@@ -28,7 +28,6 @@ Startup Ozone cluster with size
 Daemons are running without error
     Is daemon running without error           ksm
     Is daemon running without error           scm
-    Is daemon running without error           namenode
     Is daemon running without error           datanode
 
 Check if datanode is connected to the scm

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
index 3323557..12022df 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
index dec863e..e06d434 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
@@ -25,13 +24,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
 OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 6843bdd..390f089 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -34,7 +34,7 @@ function hadoop_usage
 
 
   hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop
jar and the required libraries"
-  hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
+  hadoop_add_subcommand "datanode" daemon "run a HDDS datanode"
   hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
   hadoop_add_subcommand "freon" client "runs an ozone data generator"
   hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with
tuning."
@@ -45,7 +45,7 @@ function hadoop_usage
   hadoop_add_subcommand "o3" client "command line interface for ozone"
   hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational
data"
   hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
-  hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
+  hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output
to ozone-site.xml in specified path"
 
@@ -68,10 +68,7 @@ function ozonecmd_case
     ;;
     datanode)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_SECURE_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
-      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
-      hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR
-      hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR
+      HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message