Mailing-List: contact commits-help@hawq.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hawq.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: iweng@apache.org
To: commits@hawq.incubator.apache.org
Date: Wed, 24 Aug 2016 01:51:59 -0000
Message-Id: <e51e5d5dfa9742c79abb42b7f77efc82@git.apache.org>
In-Reply-To: <1c4a52c87f794cdcbbb1e0f9534478f2@git.apache.org>
References: <1c4a52c87f794cdcbbb1e0f9534478f2@git.apache.org>
Subject: [3/3] incubator-hawq git commit: HAWQ-969. Add getting configuration
 from HDFS and YARN
archived-at: Wed, 24 Aug 2016 01:52:06 -0000

HAWQ-969. Add getting configuration from HDFS and YARN


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d1aafe68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d1aafe68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d1aafe68

Branch: refs/heads/master
Commit: d1aafe6855118e6bfcd841793b6040180ccc83b5
Parents: 8cc4a04
Author: Chunling Wang <wangchunling14@126.com>
Authored: Mon Aug 1 14:59:17 2016 +0800
Committer: ivan <iweng@pivotal.io>
Committed: Wed Aug 24 09:50:40 2016 +0800

----------------------------------------------------------------------
 src/test/feature/lib/hdfs_config.cpp  | 284 +++++++++++++++++++++++++++++
 src/test/feature/lib/hdfs_config.h    | 172 +++++++++++++++++
 src/test/feature/lib/xml_parser.cpp   | 116 +++++++++++-
 src/test/feature/lib/xml_parser.h     |  19 ++
 src/test/feature/lib/yarn_config.cpp  | 267 +++++++++++++++++++++++++++
 src/test/feature/lib/yarn_config.h    | 153 ++++++++++++++++
 src/test/feature/testlib/test_lib.cpp |  65 +++++++
 7 files changed, 1066 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/hdfs_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.cpp b/src/test/feature/lib/hdfs_config.cpp
new file mode 100644
index 0000000..90b1032
--- /dev/null
+++ b/src/test/feature/lib/hdfs_config.cpp
@@ -0,0 +1,284 @@
+#include <fstream>
+#include <string>
+#include <vector>
+#include <unordered_set>
+
+#include "hdfs_config.h"
+#include "command.h"
+#include "psql.h"
+#include "xml_parser.h"
+#include "string_util.h"
+
+using std::string;
+
+namespace hawq {
+namespace test {
+
+string HdfsConfig::getHdfsUser() {
+  string cmd = "ps aux|grep hdfs.server|grep -v grep";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  if (lines.size() >= 1) {
+    return hawq::test::trim(hawq::test::split(lines[lines.size()-1], ' ')[0]);
+  } 
+  return "hdfs";
+}
+
+bool HdfsConfig::LoadFromHawqConfigFile() {
+  const char *env = getenv("GPHOME");
+  string confPath = env ? env : "";
+  if (!confPath.empty()) {
+    confPath.append("/etc/hdfs-client.xml");
+  } else {
+    return false;
+  }
+
+  hawqxmlconf.reset(new XmlConfig(confPath));
+  hawqxmlconf->parse();
+  return true;
+}
+
+bool HdfsConfig::LoadFromHdfsConfigFile() {
+  string confPath=getHadoopHome();
+  if (confPath == "")
+    return false;
+  confPath.append("/etc/hadoop/hdfs-site.xml");
+  hdfsxmlconf.reset(new XmlConfig(confPath));
+  hdfsxmlconf->parse();
+  return true;
+}
+
+bool HdfsConfig::isHA() {
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+  string nameservice = hawqxmlconf->getString("dfs.nameservices");
+  if (nameservice.length() > 0) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool HdfsConfig::isKerbos() {
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+  string authentication = hawqxmlconf->getString("hadoop.security.authentication");
+  if (authentication == "kerberos") {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool HdfsConfig::isTruncate() {
+  string cmd = "hadoop fs -truncate";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  if (lines.size() >= 1) {
+    string valueLine = lines[0];
+    int find = valueLine.find("-truncate: Unknown command");
+    if (find < 0)
+      return true;
+  }
+  return false;
+}
+
+string HdfsConfig::getHadoopHome() {
+  string cmd = "ps -ef|grep hadoop";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  string hadoopHome = "";
+  auto lines = hawq::test::split(result, '\n');
+  for (size_t i=0; i<lines.size()-1; i++) {
+    string valueLine = lines[i];
+    int pos = valueLine.find("-Dhadoop.home.dir=");
+    if (pos >=0 ) {
+      string valueTmp = valueLine.substr(pos+18); 
+      int valueEnd = valueTmp.find_first_of(" ");
+      string value = valueTmp.substr(0, valueEnd);
+      hadoopHome = hawq::test::trim(value);
+      return hadoopHome;
+    }
+  }
+  return hadoopHome;
+}
+
+bool HdfsConfig::getActiveNamenode(string &activenamenode,
+                                   int &port) {
+    return getHANamenode("active", activenamenode, port);
+}
+
+bool HdfsConfig::getStandbyNamenode(string &standbynamenode,
+                                    int &port) {
+    return getHANamenode("standby", standbynamenode, port);
+}
+
+bool HdfsConfig::getHANamenode(const string &namenodetype,
+                               string &namenode,
+                               int &port) {
+  if (!isHA())
+    return false;
+  string namenodeService = "";
+  string nameServiceValue = hawqxmlconf->getString("dfs.nameservices");
+  string haNamenodesName = "dfs.ha.namenodes.";
+  haNamenodesName.append(hawq::test::trim(nameServiceValue));
+  string haNamenodesValue = hawqxmlconf->getString(haNamenodesName);
+  auto haNamenodes = hawq::test::split(haNamenodesValue, ',');
+  for (size_t i = 0; i < haNamenodes.size(); i++) {
+    string haNamenode = hawq::test::trim(haNamenodes[i]);
+    string cmd = "sudo -u ";
+    cmd.append(getHdfsUser());
+    cmd.append(" hdfs haadmin -getServiceState ");
+    cmd.append(haNamenode);
+    Command c(cmd);
+    string result = c.run().getResultOutput();
+    auto lines = hawq::test::split(result, '\n');
+    if (lines.size() >= 1) {
+      string valueLine = lines[0];
+      if (valueLine == namenodetype) {
+        namenodeService = haNamenode;
+        break;
+      }
+    }
+  }
+  string rpcAddressName = "dfs.namenode.rpc-address.gphd-cluster.";
+  rpcAddressName.append(namenodeService);
+  string rpcAddressValue = hawqxmlconf->getString(rpcAddressName);
+  auto namenodeInfo = hawq::test::split(rpcAddressValue, ':');
+  namenode = hawq::test::trim(namenodeInfo[0]);
+  port = std::stoi(hawq::test::trim(namenodeInfo[1]));
+  return true;
+}
+
+void HdfsConfig::getNamenodes(std::vector<string> &namenodes,
+                              std::vector<int> &port) {
+  string cmd = "sudo -u ";
+  cmd.append(getHdfsUser());
+  cmd.append(" hdfs getconf -nnRpcAddresses");
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  for (size_t i = 0; i < lines.size(); i++) {
+    string valueLine = lines[i];
+    auto namenodeInfo = hawq::test::split(valueLine, ':');
+    if (namenodeInfo.size() == 2) {
+      namenodes.push_back(hawq::test::trim(namenodeInfo[0]));
+      port.push_back(std::stoi(hawq::test::trim(namenodeInfo[1])));
+    }
+  }
+}
+
+void HdfsConfig::getDatanodelist(std::vector<string> &datanodelist,
+                                 std::vector<int> &port) {
+  string cmd = "sudo -u ";
+  cmd.append(getHdfsUser());
+  cmd.append(" hdfs dfsadmin -report | grep Name");
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  for (size_t i = 0; i < lines.size(); i++) {
+    string valueLine = lines[i];
+    auto datanodeInfo = hawq::test::split(valueLine, ':');
+    if (datanodeInfo.size() == 3) {
+      int portStart = datanodeInfo[2].find_first_of('(');
+      int portEnd = datanodeInfo[2].find_first_of(')');
+      string datanodePort = datanodeInfo[2].substr(0, portStart);
+      string datanodeHost = datanodeInfo[2].substr(portStart+1, portEnd-portStart-1);
+      datanodelist.push_back(hawq::test::trim(datanodeHost));
+      port.push_back(std::stoi(hawq::test::trim(datanodePort)));
+    }
+  }
+}
+
+void HdfsConfig::getActiveDatanodes(std::vector<string> &activedatanodes,
+                                    std::vector<int> &port) {
+  string cmd = "sudo -u ";
+  cmd.append(getHdfsUser());
+  cmd.append(" hdfs dfsadmin -report -live | grep Name");
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  for (size_t i = 0; i < lines.size(); i++) {
+    string valueLine = lines[i];
+    auto datanodeInfo = hawq::test::split(valueLine, ':');
+    if (datanodeInfo.size() == 3) {
+      int portStart = datanodeInfo[2].find_first_of('(');
+      int portEnd = datanodeInfo[2].find_first_of(')');
+      string datanodePort = datanodeInfo[2].substr(0, portStart);
+      string datanodeHost = datanodeInfo[2].substr(portStart+1, portEnd-portStart-1);
+      activedatanodes.push_back(hawq::test::trim(datanodeHost));
+      port.push_back(std::stoi(hawq::test::trim(datanodePort)));
+    }
+  }
+}
+
+
+bool HdfsConfig::isSafemode() {
+  string cmd = "hadoop fs -mkdir /tmp_hawq_test";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  if (lines.size() >= 1) {
+    string valueLine = lines[0];
+    int find = valueLine.find("Name node is in safe mode.");
+    if (find >= 0)
+      return true;
+  }
+  cmd = "hadoop fs -rm -r /tmp_hawq_test";
+  Command c_teardown(cmd);
+  result = c_teardown.run().getResultOutput();
+  return false;
+}
+
+string HdfsConfig::getParameterValue(const string &parameterName) {
+  bool ret = LoadFromHdfsConfigFile();
+  if (!ret) {
+    return NULL;
+  }
+
+  return hdfsxmlconf->getString(parameterName);
+}
+
+string HdfsConfig::getParameterValue(const string &parameterName,
+                                     const string &conftype) {
+  if (conftype == "hdfs" || conftype == "HDFS")
+    return getParameterValue(parameterName);
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return NULL;
+  }
+
+  return hawqxmlconf->getString(parameterName);
+}
+
+bool HdfsConfig::setParameterValue(const string &parameterName,
+                                   const string &parameterValue) { 
+  bool ret = LoadFromHdfsConfigFile();
+  if (!ret) {
+    return false;
+  }
+
+  return hdfsxmlconf->setString(parameterName, parameterValue);
+}
+
+bool HdfsConfig::setParameterValue(const string &parameterName,
+                                   const string &parameterValue,
+                                   const string &conftype) {
+  if (conftype == "hdfs" || conftype == "HDFS")
+    return setParameterValue(parameterName, parameterValue);
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+
+  return hawqxmlconf->setString(parameterName, parameterValue);
+}
+
+} // namespace test
+} // namespace hawq

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/hdfs_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/hdfs_config.h b/src/test/feature/lib/hdfs_config.h
new file mode 100644
index 0000000..2ea89f5
--- /dev/null
+++ b/src/test/feature/lib/hdfs_config.h
@@ -0,0 +1,172 @@
+#ifndef HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+#define HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_
+
+#include <string>
+#include <vector>
+
+#include "psql.h"
+#include "sql_util.h"
+#include "xml_parser.h"
+
+namespace hawq {
+namespace test {
+
+/**
+ * HdfsConfig common libray. Get detailed information about HDFS
+ * including checking state of namenodes and datanodes, get parameter value
+ * @author Chunling Wang
+ */
+class HdfsConfig {
+  public:
+    /**
+     * HdfsConfig constructor
+     */
+    HdfsConfig(): psql(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD) {}
+
+    /**
+     * HdfsConfig desstructor
+     */
+    ~HdfsConfig() {}
+
+    /**
+     * whether HDFS is in HA mode
+     * @return true if HDFS is HA
+     */
+    bool isHA();
+
+    /**
+     * whether HDFS is kerbos
+     * @return true if HDFS is kerbos
+     */
+    bool isKerbos();
+
+    /**
+     * whether HDFS supports truncate operation
+     * @return true if HDFS supports truncate operation
+     */
+    bool isTruncate();
+
+    /**
+     * get HADOOP working directory
+     * @return HADOOP working directory
+     */
+    std::string getHadoopHome();
+
+    /**
+     * get HDFS active namenode's hostname and port information
+     * @param activenamenode, active namenode hostname reference which will be set
+     * @param port, active namenode port reference which will be set
+     * @return true if getActiveNamenode succeeded
+     */
+    bool getActiveNamenode(std::string &activenamenode,
+                           int &port);
+
+    /**
+     * get HDFS standby namenode's hostname and port information
+     * @param standbynamenode, standby namenode hostname reference which will be set
+     * @param port, standby namenode port reference which will be set
+     * @return true if getStandbyNamenode succeeded
+     */
+    bool getStandbyNamenode(std::string &standbynamenode,
+                            int &port);
+
+    /**
+     * get HDFS namenode(s) information
+     * @param namenodes, namenodes' hostnames reference which will be set
+     * @param port, namenodes' ports reference which will be set
+     */
+    void getNamenodes(std::vector<std::string> &namenodes,
+                      std::vector<int> &port);
+
+    /**
+     * get HDFS datanodes information
+     * @param datanodelist, datanodes' hostnames reference which will be set
+     * @param port, datanodes' ports reference which will be set
+     */
+    void getDatanodelist(std::vector<std::string> &datanodelist,
+                         std::vector<int> &port);
+
+    /**
+     * get HDFS active datanodes information
+     * @param activedatanodes, active datanodes' hostnames reference which will be set
+     * @param port, active datanodes' ports reference which will be set
+     */
+    void getActiveDatanodes(std::vector<std::string> &activedatanodes,
+                            std::vector<int> &port);
+
+    /**
+     * whether HDFS is in safe mode
+     * @return true if HDFS is in safe node
+     */
+    bool isSafemode();
+
+    /**
+     * get parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, used to get parameter value
+     * @param conftype, get parameter value, 'hdfs' or 'HDFS' from ./etc/hdfs-client.xml, others from ./etc/hadoop/hdfs-site.xml
+     * @return parameter value
+     */
+    std::string getParameterValue(const std::string &parameterName, const std::string &conftype);
+    
+    /**
+     * get parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, used to get parameter value
+     * @return parameter value
+     */
+    std::string getParameterValue(const std::string &parameterName);
+
+    /**
+     * set parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, parameter name which used to set parameter value
+     * @param parameterValue, parameter value which to be set
+     * @param conftype, get parameter value, 'hdfs' or 'HDFS' from ./etc/hdfs-client.xml, others from ./etc/hadoop/hdfs-site.xml
+     * @return true if succeeded
+     */
+    bool setParameterValue(const std::string &parameterName, const std::string &parameterValue, const std::string &conftype);
+
+    /**
+     * set parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, parameter name which used to set parameter value
+     * @param parameterValue, parameter value which to be set
+     * @return true if succeeded
+     */
+    bool setParameterValue(const std::string &parameterName, const std::string &parameterValue);
+
+  private:
+    /**
+     * @return yarn user
+     */
+    std::string getHdfsUser();
+    /**
+     * load key-value parameters in ./etc/hdfs-client.xml
+     * @return true if succeeded
+     */
+    bool LoadFromHawqConfigFile();
+
+    /**
+     * load key-value parameters in ./etc/hadoop/hdfs-site.xml
+     * @return true if succeeded
+     */
+    bool LoadFromHdfsConfigFile();
+
+    /**
+     * get HDFS active or standby namenode information in HA mode according to the namenodetype
+     * @param namenodetype, used to specify active or standby namenode information
+     * @param namenode, namenode hostname reference which will be set
+     * @param port, namenode port reference which will be set
+     * @return true if getHANamenode succeeded
+     */
+    bool getHANamenode(const std::string &namenodetype,
+                       std::string &namenode,
+                                   int &port);
+
+  private:
+    std::unique_ptr<XmlConfig> hawqxmlconf;
+    std::unique_ptr<XmlConfig> hdfsxmlconf;
+    hawq::test::PSQL psql;
+};
+
+} // namespace test
+} // namespace hawq
+
+#endif /* HAWQ_SRC_TEST_FEATURE_LIB_HDFS_CONFIG_H_ */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/xml_parser.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml_parser.cpp b/src/test/feature/lib/xml_parser.cpp
index bd1edd0..3cdc229 100644
--- a/src/test/feature/lib/xml_parser.cpp
+++ b/src/test/feature/lib/xml_parser.cpp
@@ -14,26 +14,40 @@ XmlConfig::XmlConfig(string p) :
   parse();
 }
 
-void XmlConfig::parse() {
-  // the result document tree
-  xmlDocPtr doc;
-  LIBXML_TEST_VERSION kv
-  .clear();
-
+bool XmlConfig::open() {
   if (access(path.c_str(), R_OK)) {
-    return;
+    return false;
   }
 
-  // parse the file
   doc = xmlReadFile(path.c_str(), nullptr, 0);
   if (doc == nullptr) {
+    return false;
+  }
+
+  return true;
+}
+
+void XmlConfig::closeNotSave() {
+    xmlFreeDoc(doc);
+}
+
+void XmlConfig::closeAndSave() {
+    xmlSaveFormatFile(path.c_str(), doc, 0); 
+    xmlFreeDoc(doc);
+}
+
+void XmlConfig::parse() {
+  LIBXML_TEST_VERSION kv
+  .clear();
+
+  if (!open()) {
     return;
   }
   try {
     readConfigItems(doc);
-    xmlFreeDoc(doc);
+    closeNotSave();
   } catch (...) {
-    xmlFreeDoc(doc);
+    closeNotSave();
   }
 }
 
@@ -91,6 +105,88 @@ void XmlConfig::readConfigItem(xmlNodePtr root) {
   }
 }
 
+bool XmlConfig::setString(const string &key,
+                          const string &value,
+                          const bool &save) {
+  bool result = false;
+
+  if (save) {
+    if (!open()) {
+      return false;
+    }
+  }
+  try {
+    result = writeConfigItem(doc, key, value);
+    if (save) {
+      closeAndSave();
+    }
+  } catch (...) {
+    if (save) {
+      closeNotSave();
+    }
+  }
+  return result;
+}
+
+bool XmlConfig::setString(const string &key,
+                          const string &value) {
+  return setString(key, value, true);
+}
+
+bool XmlConfig::writeConfigItem(xmlDocPtr doc,
+                                const string &key,
+                                const string &value) {
+  xmlNodePtr root, curNode, curNodeChild, findNode;
+  root = xmlDocGetRootElement(doc);
+  bool findkey = false;
+  if (root == nullptr || strcmp((const char *) root->name, "configuration")) {
+    return false;
+  }
+  // for each property
+  for (curNode = root->children; curNode != nullptr; curNode = curNode->next) {
+    if (curNode->type != XML_ELEMENT_NODE) {
+      continue;
+    }
+    if (strcmp((const char *) curNode->name, "property")) {
+      return false;
+    }
+    curNodeChild = curNode->children;
+    for (curNodeChild = curNode->children; curNodeChild != nullptr; curNodeChild = curNodeChild->next) {
+      if (curNodeChild->type != XML_ELEMENT_NODE) {
+        continue;
+      }
+      if (!strcmp((const char *) curNodeChild->name, "name")) {
+        if (curNodeChild->children != nullptr
+            && XML_TEXT_NODE == curNodeChild->children->type
+            && !strcmp((const char *) curNodeChild->children->content, key.c_str())) {
+          findNode = curNode;
+          findkey = true;
+        } else {
+          if (findkey) {
+            xmlNewTextChild(findNode, NULL, BAD_CAST "value", BAD_CAST value.c_str());
+            return true;
+          }
+        }
+      } else if (!strcmp((const char *) curNodeChild->name, "value")) {
+        if (findkey
+            && curNodeChild->children != nullptr
+            && XML_TEXT_NODE == curNodeChild->children->type) {
+          xmlNodeSetContent(curNodeChild->children, (xmlChar*) const_cast<char*>(value.c_str()));
+          return true;
+        }
+      } else {
+        continue;
+      }
+    }
+  }
+
+  xmlNodePtr newNode = xmlNewNode(NULL, BAD_CAST "property");
+  xmlAddChild(root, newNode);
+  xmlNewTextChild(newNode, NULL, BAD_CAST "name", BAD_CAST key.c_str());
+  xmlNewTextChild(newNode, NULL, BAD_CAST "value", BAD_CAST value.c_str());
+  return true;
+}
+
 const string XmlConfig::getString(const char *key) {
   XmlConfigMapIterator it = kv.find(key);
   if (kv.end() == it) {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/xml_parser.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/xml_parser.h b/src/test/feature/lib/xml_parser.h
index 6978218..dde3b34 100644
--- a/src/test/feature/lib/xml_parser.h
+++ b/src/test/feature/lib/xml_parser.h
@@ -17,10 +17,27 @@ class XmlConfig {
  public:
   explicit XmlConfig(std::string);
   
+  // read an XML file into a tree
+  bool open();
+
+  // only free the XML document pointer
+  void closeNotSave();
+
+  // save the updated document to disk and free the XML document pointer
+  void closeAndSave();
+
   // parse the configuration file
   void parse();
 
   // @param key The key of the configuration item
+  // @param value The updated value
+  // @param save whether save the updated document to disk, if save is false, open() and closeAndSave() should be called additionally
+  // @ return The value of configuration item
+  bool setString(const std::string &key, const std::string &value, const bool &save);
+  
+  bool setString(const std::string &, const std::string &);
+
+  // @param key The key of the configuration item
   // @ def The default value
   // @ return The value of configuration item
   const std::string getString(const char*);
@@ -52,6 +69,7 @@ class XmlConfig {
  private:
   void readConfigItems(xmlDocPtr doc);
   void readConfigItem(xmlNodePtr root);
+  bool writeConfigItem(xmlDocPtr , const std::string &, const std::string &);
   int64_t strToInt64(const char *);
   int32_t strToInt32(const char *);
   bool strToBool(const char *);
@@ -60,6 +78,7 @@ class XmlConfig {
  private:
   std::string path;
   XmlConfigMap kv;  // key2Value
+  xmlDocPtr doc;
 }; // class XmlConfig
 
 } // namespace test

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/yarn_config.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/yarn_config.cpp b/src/test/feature/lib/yarn_config.cpp
new file mode 100644
index 0000000..79211be
--- /dev/null
+++ b/src/test/feature/lib/yarn_config.cpp
@@ -0,0 +1,267 @@
+#include <fstream>
+#include <string>
+#include <vector>
+#include <unordered_set>
+
+#include "yarn_config.h"
+#include "command.h"
+#include "psql.h"
+#include "xml_parser.h"
+#include "string_util.h"
+
+using std::string;
+
+namespace hawq {
+namespace test {
+
+string YarnConfig::getYarnUser() {
+  string cmd = "ps aux|grep yarn.server|grep -v grep";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  if (lines.size() >= 1) {
+    return hawq::test::trim(hawq::test::split(lines[lines.size()-1], ' ')[0]);
+  } 
+  return "yarn";
+}
+
+bool YarnConfig::LoadFromHawqConfigFile() {
+  const char *env = getenv("GPHOME");
+  string confPath = env ? env : "";
+  if (!confPath.empty()) {
+    confPath.append("/etc/yarn-client.xml");
+  } else {
+    return false;
+  }
+
+  hawqxmlconf.reset(new XmlConfig(confPath));
+  hawqxmlconf->parse();
+  return true;
+}
+
+bool YarnConfig::LoadFromYarnConfigFile() {
+  string confPath=getHadoopHome();
+  if (confPath == "")
+    return false;
+  confPath.append("/etc/hadoop/yarn-site.xml");
+  yarnxmlconf.reset(new XmlConfig(confPath));
+  yarnxmlconf->parse();
+  return true;
+}
+
+bool YarnConfig::isHA() {
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+  string nameservice = hawqxmlconf->getString("yarn.resourcemanager.ha");
+  if (nameservice.length() > 0) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool YarnConfig::isKerbos() {
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+  string authentication = hawqxmlconf->getString("hadoop.security.authentication");
+  if (authentication == "kerberos") {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+string YarnConfig::getHadoopHome() {
+  string cmd = "ps -ef|grep hadoop";
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  string hadoopHome = "";
+  auto lines = hawq::test::split(result, '\n');
+  for (size_t i=0; i<lines.size()-1; i++) {
+    string valueLine = lines[i];
+    int pos = valueLine.find("-Dhadoop.home.dir=");
+    if (pos >=0 ) {
+      string valueTmp = valueLine.substr(pos+18); 
+      int valueEnd = valueTmp.find_first_of(" ");
+      string value = valueTmp.substr(0, valueEnd);
+      hadoopHome = hawq::test::trim(value);
+      return hadoopHome;
+    }
+  }
+  return hadoopHome;
+}
+
+bool YarnConfig::getActiveRM(string &activeRM,
+                                   int &port) {
+    return getHARM("active", activeRM, port);
+}
+
+bool YarnConfig::getStandbyRM(string &standbyRM,
+                                    int &port) {
+    return getHARM("standby", standbyRM, port);
+}
+
+bool YarnConfig::getHARM(const string &RMtype,
+                               string &RM,
+                               int &port) {
+  if (!isHA())
+    return false;
+  string RMService = "";
+  string haRMValue = "rm1,rm2";
+  auto haRMs = hawq::test::split(haRMValue, ',');
+  for (size_t i = 0; i < haRMs.size(); i++) {
+    string haRM = hawq::test::trim(haRMs[i]);
+    string cmd ="sudo -u ";
+    cmd.append(getYarnUser());
+    cmd.append(" yarn rmadmin -getServiceState ");
+    cmd.append(haRM);
+    Command c(cmd);
+    string result = c.run().getResultOutput();
+    auto lines = hawq::test::split(result, '\n');
+    if (lines.size() >= 2) {
+      string valueLine = lines[1];
+      if (valueLine == RMtype) {
+        RMService = haRM;
+        break;
+      }
+    }
+  }
+  bool ret = LoadFromYarnConfigFile();
+  if (!ret) {
+    return false;
+  }
+  string rpcAddressName = "yarn.resourcemanager.address.";
+  rpcAddressName.append(RMService);
+  string rpcAddressValue = yarnxmlconf->getString(rpcAddressName);
+  auto RMInfo = hawq::test::split(rpcAddressValue, ':');
+  RM = hawq::test::trim(RMInfo[0]);
+  port = std::stoi(hawq::test::trim(RMInfo[1]));
+  return true;
+}
+
+bool YarnConfig::getRMList(std::vector<string> &RMList,
+                           std::vector<int> &ports){
+  string RM = "";
+  int port;
+  if (isHA()) {
+    getActiveRM(RM, port);
+    RMList.push_back(RM);
+    ports.push_back(port);
+    getStandbyRM(RM, port);
+    RMList.push_back(RM);
+    ports.push_back(port);
+    return true;
+  }
+
+  bool ret = LoadFromYarnConfigFile();
+  if (!ret) {
+    return false;
+  }
+
+  string RMAddressName = "yarn.resourcemanager.address";
+  string RMAddressValue = yarnxmlconf->getString(RMAddressName);
+  if (RMAddressValue == "")
+    return false;
+  auto RMInfo = hawq::test::split(RMAddressValue, ':');
+  RM = hawq::test::trim(RMInfo[0]);
+  port = std::stoi(hawq::test::trim(RMInfo[1]));
+  RMList.push_back(RM);
+  ports.push_back(port);
+  return true;
+}
+
+void YarnConfig::getNodeManagers(std::vector<string> &nodemanagers,
+                                 std::vector<int> &port) {
+  string cmd = "sudo -u ";
+  cmd.append(getYarnUser());
+  cmd.append(" yarn node -list -all");
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  bool begin = false;
+  for (size_t i=0; i<lines.size(); i++) {
+    if (!begin) {
+      if (lines[i].find("Node-Id") != string::npos) {
+        begin = true;
+      }
+    } else {
+      string values = hawq::test::split(lines[i], '\t')[0];
+      nodemanagers.push_back(hawq::test::trim(hawq::test::split(values, ':')[0]));
+      port.push_back(std::stoi(hawq::test::trim(hawq::test::split(values, ':')[1])));
+    }
+  }
+}
+
+void YarnConfig::getActiveNodeManagers(std::vector<string> &nodemanagers,
+                                 std::vector<int> &port) {
+  string cmd = "sudo -u ";
+  cmd.append(getYarnUser());
+  cmd.append(" yarn node -list -states RUNNING");
+  Command c(cmd);
+  string result = c.run().getResultOutput();
+  auto lines = hawq::test::split(result, '\n');
+  bool begin = false;
+  for (size_t i=0; i<lines.size(); i++) {
+    if (!begin) {
+      if (lines[i].find("Node-Id") != string::npos) {
+        begin = true;
+      }
+    } else {
+      string values = hawq::test::split(lines[i], '\t')[0];
+      nodemanagers.push_back(hawq::test::trim(hawq::test::split(values, ':')[0]));
+      port.push_back(std::stoi(hawq::test::trim(hawq::test::split(values, ':')[1])));
+    }
+  }
+}
+
+
+string YarnConfig::getParameterValue(const string &parameterName) {
+  bool ret = LoadFromYarnConfigFile();
+  if (!ret) {
+    return NULL;
+  }
+
+  return yarnxmlconf->getString(parameterName);
+}
+
+string YarnConfig::getParameterValue(const string &parameterName,
+                                     const string &conftype) {
+  if (conftype == "yarn" || conftype == "YARN")
+    return getParameterValue(parameterName);
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return NULL;
+  }
+
+  return hawqxmlconf->getString(parameterName);
+}
+
+bool YarnConfig::setParameterValue(const string &parameterName,
+                                   const string &parameterValue) {
+  bool ret = LoadFromYarnConfigFile();
+  if (!ret) {
+    return false;
+  }
+
+  return yarnxmlconf->setString(parameterName, parameterValue);
+}
+
+bool YarnConfig::setParameterValue(const string &parameterName,
+                                   const string &parameterValue,
+                                   const string &conftype) {
+  if (conftype == "yarn" || conftype == "YARN")
+    return setParameterValue(parameterName, parameterValue);
+  bool ret = LoadFromHawqConfigFile();
+  if (!ret) {
+    return false;
+  }
+
+  return hawqxmlconf->setString(parameterName, parameterValue);
+}
+
+} // namespace test
+} // namespace hawq

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/lib/yarn_config.h
----------------------------------------------------------------------
diff --git a/src/test/feature/lib/yarn_config.h b/src/test/feature/lib/yarn_config.h
new file mode 100644
index 0000000..559daca
--- /dev/null
+++ b/src/test/feature/lib/yarn_config.h
@@ -0,0 +1,153 @@
+#ifndef HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_
+#define HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_
+
+#include <string>
+#include <vector>
+
+#include "psql.h"
+#include "sql_util.h"
+#include "xml_parser.h"
+
+namespace hawq {
+namespace test {
+
+/**
+ * YarnConfig common libray. Get detailed information about YARN
+ * including checking state of resourcemanagers and nodemanagers, get parameter value
+ * @author Chunling Wang
+ */
+class YarnConfig {
+  public:
+    /**
+      * YarnConfig constructor
+      */
+    YarnConfig(): psql(HAWQ_DB, HAWQ_HOST, HAWQ_PORT, HAWQ_USER, HAWQ_PASSWORD) {}
+
+    /**
+      * YarnConfig desstructor
+      */
+    ~YarnConfig()  {}
+
+    /**
+     * whether YARN is in HA mode
+     * @return true if YARN is HA
+     */
+    bool isHA();
+
+    /**
+     * whether YARN is kerbos
+     * @return true if YARN is kerbos
+     */
+    bool isKerbos();
+
+    /**
+     * get HADOOP working directory
+     * @return HADOOP working directory
+     */
+    std::string getHadoopHome();
+
+    /**
+     * get YARN active resourcemanager's hostname and port information
+     * @param activeresourcemanager, active resourcemanager hostname reference which will be set
+     * @param port, active resourcemanager port reference which will be set
+     * @return true if getActiveRM succeeded
+     */
+    bool getActiveRM(std::string &activeRM, int &port);
+
+    /**
+     * get YARN standby resourcemanager's hostname and port information
+     * @param standbyRM, standby resourcemanager hostname reference which will be set
+     * @param port, standby resourcemanager port reference which will be set
+     * @return true if getStandbyRM succeeded
+     */
+    bool getStandbyRM(std::string &standbyRM, int &port);
+
+    /**
+     * get YARN resourcemanager(s) information
+     * @param RMList, resourcemanagers' hostnames reference which will be set
+     * @param port, resourcemanagers' ports reference which will be set
+     */
+    bool getRMList(std::vector<std::string> &RMList, std::vector<int> &port);
+
+    /**
+     * get YARN nodemanagers information
+     * @param nodemanagers, nodemanagers' hostnames reference which will be set
+     * @param port, nodemanagers' ports reference which will be set
+     */
+    void getNodeManagers(std::vector<std::string> &nodemanagers, std::vector<int> &port);
+
+    /**
+     * get YARN active nodemanagers information
+     * @param nodemanagers, active nodemanagers' hostnames reference which will be set
+     * @param port, active nodemanagers' ports reference which will be set
+     */
+    void getActiveNodeManagers(std::vector<std::string> &nodemanagers, std::vector<int> &port);
+
+    /**
+     * get parameter value in ./etc/yarn-client.xml or ./etc/hadoop/yarn-site.xml according to parameter name
+     * @param parameterName, used to get parameter value
+     * @param conftype, get parameter value, 'yarn' or 'YARN' from ./etc/yarn-client.xml, others from ./etc/hadoop/yarn-site.xml
+     * @return parameter value
+     */
+    std::string getParameterValue(const std::string &parameterName);
+
+    /**
+     * get parameter value in ./etc/hadoop/yarn-site.xml according to parameter name
+     * @param parameterName, used to get parameter value
+     * @return parameter value
+     */
+    std::string getParameterValue(const std::string &parameterName, const std::string &conftype);
+
+    /**
+     * set parameter value in ./etc/hdfs-client.xml or ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, parameter name which used to set parameter value
+     * @param parameterValue, parameter value which to be set
+     * @param conftype, get parameter value, 'yarn' or 'YARN' from ./etc/yarn-client.xml, others from ./etc/hadoop/yarn-site.xml
+     * @return true if succeeded
+     */
+    bool setParameterValue(const std::string &parameterName, const std::string &parameterValue);
+
+    /**
+     * set parameter value in ./etc/hadoop/hdfs-site.xml according to parameter name
+     * @param parameterName, parameter name which used to set parameter value
+     * @param parameterValue, parameter value which to be set
+     * @return true if succeeded
+     */
+    bool setParameterValue(const std::string &parameterName, const std::string &parameterValue, const std::string &conftype);
+
+  private:
+    /**
+     * @return yarn user
+     */
+    std::string getYarnUser();
+    /**
+     * load key-value parameters in ./etc/yarn-client.xml
+     * @return true if succeeded
+     */
+    bool LoadFromHawqConfigFile();
+
+    /**
+     * load key-value parameters in ./etc/hadoop/yarn-site.xml
+     * @return true if succeeded
+     */
+    bool LoadFromYarnConfigFile();
+
+    /**
+     * get Yarn active or standby resourcemanager information in HA mode according to the RMtype
+     * @param RMtype, used to specify active or standby resourcemanager information
+     * @param RM, resourcemanager hostname reference which will be set
+     * @param port, resourcemanager port reference which will be set
+     * @return true if getHARM succeeded
+     */
+    bool getHARM(const std::string &RMtype, std::string &RMnode, int &port);
+
+  private:
+    std::unique_ptr<XmlConfig> hawqxmlconf;
+    std::unique_ptr<XmlConfig> yarnxmlconf;
+    hawq::test::PSQL psql;
+};
+
+} // namespace test
+} // namespace hawq
+
+#endif /* HAWQ_SRC_TEST_FEATURE_LIB_YARN_CONFIG_H_ */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d1aafe68/src/test/feature/testlib/test_lib.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/testlib/test_lib.cpp b/src/test/feature/testlib/test_lib.cpp
index 98b90d2..a577c93 100644
--- a/src/test/feature/testlib/test_lib.cpp
+++ b/src/test/feature/testlib/test_lib.cpp
@@ -9,6 +9,8 @@
 #include "lib/command.h"
 #include "lib/data_gen.h"
 #include "lib/hawq_config.h"
+#include "lib/hdfs_config.h"
+#include "lib/yarn_config.h"
 #include "lib/sql_util.h"
 #include "lib/string_util.h"
 #include "lib/file_replace.h"
@@ -56,6 +58,69 @@ TEST_F(TestCommonLib, TestHawqConfig) {
   return;
 }
 
+TEST_F(TestCommonLib, TestHdfsConfig) {
+  hawq::test::HdfsConfig hc;
+  hc.isHA();
+  hc.isKerbos();
+  hc.isTruncate();
+  std::string hadoopHome = hc.getHadoopHome();
+
+  std::string hostname = "";
+  int port = 0;
+  hc.getActiveNamenode(hostname, port);
+
+  hostname = "";
+  port = 0;
+  hc.getStandbyNamenode(hostname, port);
+
+  std::vector<std::string> hostList;
+  std::vector<int> portList;
+  hc.getNamenodes(hostList, portList);
+
+  hostList.clear();
+  portList.clear();
+  hc.getDatanodelist(hostList, portList);
+
+  hostList.clear();
+  portList.clear();
+  hc.getActiveDatanodes(hostList, portList);
+
+  hc.isSafemode();
+
+  hc.getParameterValue("dfs.replication");
+  hc.setParameterValue("dfs.replication", "1");
+}
+
+TEST_F(TestCommonLib, TestYarnConfig) {
+  hawq::test::YarnConfig hc;
+  hc.isHA();
+  hc.isKerbos();
+  std::string hadoopHome = hc.getHadoopHome();
+
+  std::string hostname = "";
+  int port = 0;
+  hc.getActiveRM(hostname, port);
+
+  hostname = "";
+  port = 0;
+  hc.getStandbyRM(hostname, port);
+
+  std::vector<std::string> hostList;
+  std::vector<int> portList;
+  hc.getRMList(hostList, portList);
+
+  hostList.clear();
+  portList.clear();
+  hc.getNodeManagers(hostList, portList);
+
+  hostList.clear();
+  portList.clear();
+  hc.getActiveNodeManagers(hostList, portList);
+
+  hc.getParameterValue("yarn.scheduler.minimum-allocation-mb");
+  hc.setParameterValue("yarn.scheduler.minimum-allocation-mb", "1024");
+}
+
 TEST_F(TestCommonLib, TestCommand) {
   hawq::test::Command c("ls ./");
   c.run().getResultOutput();