hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [hadoop-ozone] 01/12: HDDS-2646. Start acceptance tests only if at least one THREE pipeline is available
Date Tue, 03 Dec 2019 12:35:38 GMT
This is an automated email from the ASF dual-hosted git repository.

elek pushed a commit to branch debug-acceptance-timeout
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git

commit 9685ec0cdb1f1ad5504765207110e0346886ea24
Author: Márton Elek <elek@apache.org>
AuthorDate: Thu Nov 28 17:06:45 2019 +0100

    HDDS-2646. Start acceptance tests only if at least one THREE pipeline is available
---
 .../dist/src/main/compose/ozone-hdfs/docker-config |  1 +
 .../dist/src/main/compose/ozone-mr/common-config   |  1 +
 .../src/main/compose/ozone-om-ha-s3/docker-config  |  1 +
 .../src/main/compose/ozone-recon/docker-config     |  1 +
 .../src/main/compose/ozone-topology/docker-config  |  2 +
 .../dist/src/main/compose/ozone/docker-config      |  1 +
 .../src/main/compose/ozoneblockade/docker-config   |  2 +
 .../dist/src/main/compose/ozoneperf/docker-config  |  1 +
 .../src/main/compose/ozones3-haproxy/docker-config |  1 +
 .../dist/src/main/compose/ozones3/docker-config    |  1 +
 .../src/main/compose/ozonescripts/docker-config    |  2 +
 .../src/main/compose/ozonesecure-mr/docker-config  |  1 +
 .../src/main/compose/ozonesecure/docker-config     |  1 +
 hadoop-ozone/dist/src/main/compose/testlib.sh      | 49 ----------------------
 14 files changed, 16 insertions(+), 49 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
index a8a96c3..c6295eb 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
@@ -23,6 +23,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=1
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
index ee7a773..14c33ee 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
index 4122dae..5cfa826 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
@@ -29,5 +29,6 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
index bf7871e..0c817a9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config
@@ -27,6 +27,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
index ec9c590..f8c939f 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
@@ -29,6 +29,8 @@ OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.con
 OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
 OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
 OZONE-SITE.XML_dfs.network.topology.aware.read.enable=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
index d82cde2..02b532a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -25,6 +25,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 ASYNC_PROFILER_HOME=/opt/profiler
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
index 3625fc6..b854aab 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
@@ -34,6 +34,8 @@ OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
 OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s
 OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
 OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
index 931f112..5d68afc 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
@@ -25,6 +25,7 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 ASYNC_PROFILER_HOME=/opt/profiler
 
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
index 75d1977..4c08c52 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
index 75d1977..4c08c52 100644
--- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config
@@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
index 9950898..787ef2e 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
@@ -26,6 +26,8 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
 OZONE-SITE.XML_ozone.replication=1
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
 OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
+
 HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
 HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index 7d6fdc4..54992d8 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -31,6 +31,7 @@ OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
 OZONE-SITE.XML_ozone.s3g.keytab.file=/etc/security/keytabs/HTTP.keytab
 OZONE-SITE.XML_ozone.s3g.authentication.kerberos.principal=HTTP/s3g@EXAMPLE.COM
 OZONE-SITE.XML_ozone.administrators=*
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 OZONE-SITE.XML_ozone.security.enabled=true
 OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index ff42249..14ef63c 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -36,6 +36,7 @@ OZONE-SITE.XML_ozone.recon.authentication.kerberos.principal=HTTP/recon@EXAMPLE.
 OZONE-SITE.XML_ozone.recon.keytab.file=/etc/security/keytabs/HTTP.keytab
 OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
 OZONE-SITE.XML_recon.om.snapshot.task.initial.delay=20s
+OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
 
 OZONE-SITE.XML_ozone.security.enabled=true
 OZONE-SITE.XML_ozone.acl.enabled=true
diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh
index 49274c0..f4b0fbc 100755
--- a/hadoop-ozone/dist/src/main/compose/testlib.sh
+++ b/hadoop-ozone/dist/src/main/compose/testlib.sh
@@ -31,54 +31,6 @@ create_results_dir() {
   chmod ogu+w "$RESULT_DIR"
 }
 
-## @description print the number of datanodes up
-## @param the docker-compose file
-count_datanodes() {
-  local compose_file=$1
-
-  local jmx_url='http://scm:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo'
-  if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
-    docker-compose -f "${compose_file}" exec -T scm bash -c "kinit -k HTTP/scm@EXAMPLE.COM
-t /etc/security/keytabs/HTTP.keytab && curl --negotiate -u : -s '${jmx_url}'"
-  else
-    docker-compose -f "${compose_file}" exec -T scm curl -s "${jmx_url}"
-  fi \
-    | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value' || true
-}
-
-## @description wait until datanodes are up (or 30 seconds)
-## @param the docker-compose file
-## @param number of datanodes to wait for (default: 3)
-wait_for_datanodes(){
-  local compose_file=$1
-  local -i datanode_count=${2:-3}
-
-  #Reset the timer
-  SECONDS=0
-
-  #Don't give it up until 30 seconds
-  while [[ $SECONDS -lt 90 ]]; do
-
-     #This line checks the number of HEALTHY datanodes registered in scm over the
-     # jmx HTTP servlet
-     datanodes=$(count_datanodes "${compose_file}")
-     if [[ "$datanodes" ]]; then
-       if [[ ${datanodes} -ge ${datanode_count} ]]; then
-
-         #It's up and running. Let's return from the function.
-         echo "$datanodes datanodes are up and registered to the scm"
-         return
-       else
-
-           #Print it only if a number. Could be not a number if scm is not yet started
-           echo "$datanodes datanode is up and healthy (until now)"
-         fi
-     fi
-
-      sleep 2
-   done
-   echo "WARNING! Datanodes are not started successfully. Please check the docker-compose
files"
-   return 1
-}
 
 ## @description wait until safemode exit (or 30 seconds)
 ## @param the docker-compose file
@@ -123,7 +75,6 @@ start_docker_env(){
 
   docker-compose -f "$COMPOSE_FILE" --no-ansi down
   docker-compose -f "$COMPOSE_FILE" --no-ansi up -d --scale datanode="${datanode_count}"
\
-    && wait_for_datanodes "$COMPOSE_FILE" "${datanode_count}" \
     && wait_for_safemode_exit "$COMPOSE_FILE" \
     && sleep 10
 


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message