ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-11006 - YARN Upgrade Pack For HDP-2.2 To HDP-2.3 (jonathanhurley)
Date Fri, 08 May 2015 03:48:43 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 1b66baac0 -> 8c3d2acd4


AMBARI-11006 - YARN Upgrade Pack For HDP-2.2 To HDP-2.3 (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8c3d2acd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8c3d2acd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8c3d2acd

Branch: refs/heads/trunk
Commit: 8c3d2acd445dde75f0c6af7f13ef874aa071603d
Parents: 1b66baa
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Thu May 7 22:44:50 2015 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Thu May 7 23:48:37 2015 -0400

----------------------------------------------------------------------
 .../serveraction/upgrades/ConfigureAction.java  | 30 ++++++++++----
 .../state/stack/upgrade/ConfigureTask.java      |  3 ++
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     | 43 +++++++++++++++++++-
 .../GLUSTERFS/configuration/hadoop-env.xml      |  2 +-
 .../HIVE/configuration/webhcat-site.xml         |  2 +-
 .../services/HDFS/configuration/hadoop-env.xml  |  4 +-
 .../ranger-yarn-plugin-properties.xml           |  4 +-
 .../upgrades/ConfigureActionTest.java           | 22 ++++++++++
 8 files changed, 95 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
index a812169..c064ac4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
@@ -42,6 +42,7 @@ import org.apache.ambari.server.state.ConfigMergeHelper.ThreeWayValue;
 import org.apache.ambari.server.state.DesiredConfig;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
+import org.apache.commons.lang.StringUtils;
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
@@ -186,9 +187,17 @@ public class ConfigureAction extends AbstractServerAction {
     for (ConfigureTask.Transfer transfer : transfers) {
       switch (transfer.operation) {
         case COPY:
-          if (null == transfer.fromType && base.containsKey(transfer.fromKey)) {
-            newValues.put(transfer.toKey, base.get(transfer.fromKey));
-            changedValues = true;
+          // if copying from the current configuration type, then first
+          // determine if the key already exists; if it does not, then set a
+          // default if a default exists
+          if (null == transfer.fromType) {
+            if (base.containsKey(transfer.fromKey)) {
+              newValues.put(transfer.toKey, base.get(transfer.fromKey));
+              changedValues = true;
+            } else if (StringUtils.isNotBlank(transfer.defaultValue)) {
+              newValues.put(transfer.toKey, transfer.defaultValue);
+              changedValues = true;
+            }
           } else {
             // !!! copying from another configuration
             Config other = cluster.getDesiredConfigByType(transfer.fromType);
@@ -199,14 +208,23 @@ public class ConfigureAction extends AbstractServerAction {
               if (otherValues.containsKey(transfer.fromKey)) {
                 newValues.put(transfer.toKey, otherValues.get(transfer.fromKey));
                 changedValues = true;
+              } else if (StringUtils.isNotBlank(transfer.defaultValue)) {
+                newValues.put(transfer.toKey, transfer.defaultValue);
+                changedValues = true;
               }
             }
           }
           break;
         case MOVE:
+          // if the value existed previously, then update the maps with the new
+          // key; otherwise if there is a default value specified, set the new
+          // key with the default
           if (newValues.containsKey(transfer.fromKey)) {
             newValues.put(transfer.toKey, newValues.remove(transfer.fromKey));
             changedValues = true;
+          } else if (StringUtils.isNotBlank(transfer.defaultValue)) {
+            newValues.put(transfer.toKey, transfer.defaultValue);
+            changedValues = true;
           }
 
           break;
@@ -261,9 +279,7 @@ public class ConfigureAction extends AbstractServerAction {
       config.persist(false);
 
       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
-          MessageFormat.format("Updated ''{0}'' with ''{1}={2}''",
-              configType, key, value),
-          "");
+          MessageFormat.format("Updated configuration ''{0}''", configType), "");
     }
 
     // !!! values are different and within the same stack.  create a new
@@ -279,7 +295,7 @@ public class ConfigureAction extends AbstractServerAction {
     m_configHelper.createConfigType(cluster, m_controller, configType,
         newValues, auditName, serviceVersionNote);
 
-    String message = "Updated ''{0}'' with ''{1}={2}''";
+    String message = "Updated configuration ''{0}'' with ''{1}={2}''";
     message = MessageFormat.format(message, configType, key, value);
 
     return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", message, "");

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index 1f921c9..2b68a28 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -193,6 +193,9 @@ public class ConfigureTask extends ServerSideActionTask {
     @XmlAttribute(name = "preserve-edits")
     public boolean preserveEdits = false;
 
+    @XmlAttribute(name = "default-value")
+    public String defaultValue;
+
     @XmlElement(name = "keep-key")
     public List<String> keepKeys = new ArrayList<String>();
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 225cc63..abd95aa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -351,6 +351,20 @@
 
     <service name="MAPREDUCE2">
       <component name="HISTORYSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure">
+            <type>mapred-site</type>
+            <transfer operation="MOVE" from-key="mapreduce.job.speculative.speculativecap"
to-key="mapreduce.job.speculative.speculative-cap-running-tasks" default-value="0.1" />
+            <transfer operation="DELETE" delete-key="mapreduce.task.tmp.dir" />
+          </task>
+
+          <task xsi:type="configure">
+            <type>mapred-site</type>
+            <key>mapreduce.fileoutputcommitter.algorithm.version</key>
+            <value>1</value>
+          </task>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>
@@ -365,12 +379,38 @@
 
     <service name="YARN">
       <component name="APP_TIMELINE_SERVER">
+        <pre-upgrade>
+          <task xsi:type="configure">
+            <type>yarn-site</type>
+            <key>yarn.timeline-service.recovery.enabled</key>
+            <value>true</value>
+          </task>
+
+          <task xsi:type="configure">
+            <type>yarn-site</type>
+            <transfer operation="COPY" from-key="yarn.timeline-service.leveldb-timeline-store.path"
to-key="yarn.timeline-service.leveldb-state-store.path"/>
+          </task>
+
+          <task xsi:type="configure">
+            <type>yarn-site</type>
+            <key>yarn.timeline-service.state-store-class</key>
+            <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
+          </task>
+        </pre-upgrade>
+
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>
       </component>
 
       <component name="RESOURCEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure">
+            <type>yarn-site</type>
+            <key>yarn.node-labels.enabled</key>
+            <value>false</value>
+          </task>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>
@@ -575,8 +615,7 @@
           </task>
           <task xsi:type="configure" summary="Setting nimbus.seeds from nimbus.host">
             <type>storm-site</type>
-            <transfer operation="COPY" from-key="nimbus.host"
-              to-type="storm-site" to-key="nimbus.seeds" />
+            <transfer operation="COPY" from-key="nimbus.host" to-key="nimbus.seeds" />
           </task>
         </pre-upgrade>
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml
index bce6b53..e9078e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/GLUSTERFS/configuration/hadoop-env.xml
@@ -180,7 +180,7 @@ MAPREDUCE_LIBS={{mapreduce_libs_path}}
 export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
 
 if [ -d "/usr/lib/tez" ]; then
-  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf
+  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/usr/hdp/current/tez-client/conf
 fi
 
 # Setting path to hdfs command line

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml
index a7caae4..4293ea5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3.GlusterFS/services/HIVE/configuration/webhcat-site.xml
@@ -31,7 +31,7 @@ limitations under the License.
 
   <property>
     <name>templeton.hadoop.conf.dir</name>
-    <value>/etc/hadoop/conf</value>
+    <value>/usr/hdp/current/hadoop-client/conf</value>
     <description>The path to the Hadoop configuration.</description>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
index cb641d6..108805f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml
@@ -140,9 +140,9 @@ export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
 
 # added to the HADOOP_CLASSPATH
 if [ -d "/usr/hdp/current/tez-client" ]; then
-  if [ -d "/etc/tez/conf/" ]; then
+  if [ -d "/usr/hdp/current/tez-client/conf/" ]; then
     # When using versioned RPMs, the tez-client will be a symlink to the current folder of
tez in HDP.
-    export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/
+    export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/
   fi
 fi
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml
index 93035d1..7fa47d9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/ranger-yarn-plugin-properties.xml
@@ -210,7 +210,7 @@
 
   <property>
     <name>SSL_KEYSTORE_FILE_PATH</name>
-    <value>/etc/hadoop/conf/ranger-plugin-keystore.jks</value>
+    <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks</value>
     <description></description>
   </property>
 
@@ -222,7 +222,7 @@
 
   <property>
     <name>SSL_TRUSTSTORE_FILE_PATH</name>
-    <value>/etc/hadoop/conf/ranger-plugin-truststore.jks</value>
+    <value>/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks</value>
     <description></description>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8c3d2acd/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
index ecd55db..d9638bc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
@@ -247,6 +247,7 @@ public class ConfigureActionTest {
     commandParams.put(ConfigureTask.PARAMETER_KEY, "initLimit");
     commandParams.put(ConfigureTask.PARAMETER_VALUE, "11");
 
+    // normal copy
     List<ConfigureTask.Transfer> transfers = new ArrayList<ConfigureTask.Transfer>();
     ConfigureTask.Transfer transfer = new ConfigureTask.Transfer();
     transfer.operation = TransferOperation.COPY;
@@ -254,12 +255,29 @@ public class ConfigureActionTest {
     transfer.toKey = "copyKey";
     transfers.add(transfer);
 
+    // copy with default
+    transfer = new ConfigureTask.Transfer();
+    transfer.operation = TransferOperation.COPY;
+    transfer.fromKey = "copiedFromMissingKeyWithDefault";
+    transfer.toKey = "copiedToMissingKeyWithDefault";
+    transfer.defaultValue = "defaultValue";
+    transfers.add(transfer);
+
+    // normal move
     transfer = new ConfigureTask.Transfer();
     transfer.operation = TransferOperation.MOVE;
     transfer.fromKey = "moveIt";
     transfer.toKey = "movedKey";
     transfers.add(transfer);
 
+    // move with default
+    transfer = new ConfigureTask.Transfer();
+    transfer.operation = TransferOperation.MOVE;
+    transfer.fromKey = "movedFromKeyMissingWithDefault";
+    transfer.toKey = "movedToMissingWithDefault";
+    transfer.defaultValue = "defaultValue2";
+    transfers.add(transfer);
+
     transfer = new ConfigureTask.Transfer();
     transfer.operation = TransferOperation.DELETE;
     transfer.deleteKey = "deleteIt";
@@ -300,6 +318,10 @@ public class ConfigureActionTest {
     assertFalse(map.containsKey("moveIt"));
     assertTrue(map.containsKey("movedKey"));
     assertFalse(map.containsKey("deletedKey"));
+    assertTrue(map.containsKey("copiedToMissingKeyWithDefault"));
+    assertEquals("defaultValue", map.get("copiedToMissingKeyWithDefault"));
+    assertTrue(map.containsKey("movedToMissingWithDefault"));
+    assertEquals("defaultValue2", map.get("movedToMissingWithDefault"));
 
     transfers.clear();
     transfer = new ConfigureTask.Transfer();


Mime
View raw message