ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [25/25] ambari git commit: Merge branch 'trunk' into branch-feature-AMBARI-18456
Date Tue, 29 Nov 2016 22:01:35 GMT
Merge branch 'trunk' into branch-feature-AMBARI-18456


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/276d1244
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/276d1244
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/276d1244

Branch: refs/heads/branch-feature-AMBARI-18456
Commit: 276d1244e1b1d8200cf644b7cccfaa0a757f8146
Parents: 1f804d1 85c9104
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Tue Nov 29 16:51:58 2016 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Tue Nov 29 17:00:59 2016 -0500

----------------------------------------------------------------------
 .../views/stackVersions/stackVersionPage.html   |   2 +-
 .../org/apache/ambari/logsearch/LogSearch.java  |  15 +-
 .../logsearch/common/ExternalServerClient.java  |  12 +-
 .../logsearch/common/LogSearchConstants.java    |   1 +
 .../AbstractOperationHolderConverter.java       |   9 +
 ...actServiceLogRequestFacetQueryConverter.java |  44 ++
 .../BaseServiceLogRequestQueryConverter.java    |   1 +
 ...ServiceLogAnyGraphRequestQueryConverter.java |  13 +
 ...eLogComponentLevelRequestQueryConverter.java |  15 +-
 ...eLogComponentRequestFacetQueryConverter.java |  15 +-
 ...rviceLogLevelCountRequestQueryConverter.java |  16 +
 ...eLogLevelDateRangeRequestQueryConverter.java |  16 +-
 ...erviceLogTreeRequestFacetQueryConverter.java |  17 +-
 .../ambari/logsearch/doc/DocConstants.java      |   1 +
 .../logsearch/graph/GraphDataGenerator.java     |  26 +-
 .../logsearch/manager/ServiceLogsManager.java   |   2 +-
 .../request/ServiceLogParamDefinition.java      |   6 +
 .../request/impl/BaseServiceLogRequest.java     |  13 +
 .../LogsearchKRBAuthenticationFilter.java       |   4 +-
 .../src/main/resources/logsearch.properties     |   2 +-
 .../dashboard/BubbleGraphTableLayoutView.js     |   4 +-
 .../views/dashboard/ComponentListView.js        |   2 +-
 .../scripts/views/dashboard/HostListView.js     |  14 +-
 .../main/webapp/scripts/views/tabs/TreeView.js  |  10 +-
 .../webapp/templates/tabs/TreeView_tmpl.html    | 115 ++--
 ...ComponentLevelRequestQueryConverterTest.java |   7 +-
 ...ComponentRequestFacetQueryConverterTest.java |   4 +-
 .../conf/unix/ambari-metrics-grafana            |  34 +-
 .../ambari/server/checks/CheckDescription.java  |  13 +
 .../server/checks/ServicePresenceCheck.java     | 177 ++++++
 .../internal/ClusterControllerImpl.java         |   2 +-
 .../ServiceConfigVersionResourceProvider.java   |   3 +-
 .../logging/LoggingSearchPropertyProvider.java  |  11 +-
 .../ambari/server/orm/dao/ClusterDAO.java       |  24 +
 .../ambari/server/orm/dao/ServiceConfigDAO.java |  13 +-
 .../orm/entities/ClusterConfigEntity.java       |   7 +-
 .../entities/ClusterConfigMappingEntity.java    |   6 +
 .../orm/entities/ServiceConfigEntity.java       |   5 +-
 .../server/orm/helpers/dbms/MySqlHelper.java    |  18 +
 .../upgrades/RangerKmsProxyConfig.java          |  94 +++
 .../server/state/cluster/ClusterImpl.java       |  42 +-
 .../server/upgrade/SchemaUpgradeHelper.java     |  35 +-
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |   1 +
 .../FLUME/1.4.0.2.0/package/scripts/flume.py    |   1 +
 .../package/files/hbaseSmokeVerify.sh           |   2 +-
 .../HDFS/2.1.0.2.0/kerberos.json                |   2 +-
 .../HIVE/0.12.0.2.0/configuration/hive-env.xml  |  19 +
 .../HIVE/0.12.0.2.0/package/scripts/webhcat.py  |  10 -
 .../configuration/logfeeder-ambari-config.xml   |  37 ++
 .../configuration/logfeeder-output-config.xml   |  37 ++
 .../configuration/logsearch-properties.xml      |  40 ++
 .../LOGSEARCH/0.5.0/metainfo.xml                |   5 +
 .../LOGSEARCH/0.5.0/package/scripts/params.py   |  17 +-
 .../0.5.0/package/scripts/setup_logfeeder.py    |  10 +
 .../templates/input.config-ambari.json.j2       | 602 -------------------
 .../package/templates/output.config.json.j2     |  61 --
 .../properties/input.config-ambari.json.j2      | 602 +++++++++++++++++++
 .../0.5.0/properties/output.config.json.j2      |  61 ++
 .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml |  19 +
 .../0.6.0.2.5/package/scripts/master.py         |  15 +-
 .../0.6.0.2.5/package/scripts/params.py         |  28 +-
 .../ZEPPELIN/0.6.0.2.5/role_command_order.json  |   2 +-
 .../HDP/2.0.6/configuration/cluster-env.xml     |   6 +
 .../HDP/2.2/services/HIVE/themes/theme.json     | 107 +++-
 .../services/HIVE/configuration/hive-env.xml    |  35 ++
 .../services/OOZIE/configuration/oozie-env.xml  |  33 +
 .../stacks/HDP/2.3/services/OOZIE/metainfo.xml  |   6 +
 .../HDP/2.3/services/OOZIE/themes/theme.json    | 116 ++++
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml |   7 +
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml |   7 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |  11 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml     |  11 +
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml |   7 +
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml |   7 +
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     |  11 +
 .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml     |  11 +
 .../stacks/HDP/2.5/services/HDFS/kerberos.json  |   2 +-
 .../stacks/HDP/2.5/services/OOZIE/metainfo.xml  |   6 +
 .../HDP/2.5/services/OOZIE/themes/theme.json    | 116 ++++
 .../HDP/2.5/upgrades/host-ordered-upgrade.xml   |  11 +-
 .../server/checks/ServicePresenceCheckTest.java | 217 +++++++
 .../upgrades/RangerKmsProxyConfigTest.java      | 125 ++++
 .../stacks/2.0.6/HIVE/test_webhcat_server.py    |   4 -
 .../stacks/2.4/LOGSEARCH/test_logfeeder.py      |  14 +-
 .../test/python/stacks/2.4/configs/default.json |   6 +
 ambari-web/app/app.js                           |   4 -
 .../main/admin/service_auto_start.js            |   2 +-
 .../controllers/main/service/add_controller.js  |  11 +-
 .../data/configs/services/hive_properties.js    |   4 -
 .../configs/services/logsearch_properties.js    |  51 +-
 .../data/configs/services/oozie_properties.js   |   4 -
 ambari-web/app/messages.js                      |   3 +
 ambari-web/app/styles/application.less          |  10 +-
 ambari-web/app/styles/bootstrap_overrides.less  |   9 +
 .../app/styles/enhanced_service_dashboard.less  | 343 ++++-------
 ambari-web/app/styles/hosts.less                |   3 -
 .../app/styles/theme/bootstrap-ambari.css       |   5 +-
 .../modal_popups/widget_browser_popup.hbs       | 108 ++--
 .../templates/main/admin/service_auto_start.hbs |   2 +-
 .../main/service/widgets/create/expression.hbs  |  36 +-
 .../main/service/widgets/create/step1.hbs       |  18 +-
 .../main/service/widgets/create/step2.hbs       |  22 +-
 .../service/widgets/create/step2_add_metric.hbs |  12 +-
 .../main/service/widgets/create/step2_graph.hbs |   4 +-
 .../service/widgets/create/step2_number.hbs     |   4 +-
 .../service/widgets/create/step2_template.hbs   |   4 +-
 .../main/service/widgets/create/step3.hbs       |  42 +-
 .../create/widget_property_threshold.hbs        |  18 +-
 ambari-web/app/views/common/chosen_plugin.js    |  21 +-
 ambari-web/app/views/common/controls_view.js    |   2 +-
 ambari-web/app/views/main/dashboard/widgets.js  |  17 +-
 .../main/service/widgets/create/step2_view.js   |   2 +
 .../main/service/add_controller_test.js         |  29 +-
 .../test/views/main/dashboard/widgets_test.js   |  11 +-
 contrib/utils/perf/deploy-gce-perf-cluster.py   | 243 +++++---
 docs/pom.xml                                    |   4 +-
 116 files changed, 2957 insertions(+), 1408 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
----------------------------------------------------------------------
diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
index 0000000,bb88f55..25387cc
mode 000000,100644..100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfig.java
@@@ -1,0 -1,95 +1,94 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.ambari.server.serveraction.upgrades;
+ 
+ import java.text.MessageFormat;
+ import java.util.Map;
+ import java.util.concurrent.ConcurrentMap;
+ 
+ import org.apache.ambari.server.AmbariException;
+ import org.apache.ambari.server.actionmanager.HostRoleStatus;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.serveraction.AbstractServerAction;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.Clusters;
+ import org.apache.ambari.server.state.Config;
+ import org.apache.ambari.server.state.SecurityType;
 -import org.apache.commons.lang.StringUtils;
+ 
+ import com.google.inject.Inject;
+ 
+ /**
+ * Computes Ranger KMS Proxy properties in kms-site
+ */
+ 
+ public class RangerKmsProxyConfig extends AbstractServerAction {
+   private static final String RANGER_ENV_CONFIG_TYPE = "ranger-env";
+   private static final String RANGER_KMS_SITE_CONFIG_TYPE = "kms-site";
+ 
+   @Inject
+   private Clusters m_clusters;
+ 
+   @Override
+   public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+     throws AmbariException, InterruptedException {
+ 
+     String clusterName = getExecutionCommand().getClusterName();
+     Cluster cluster = m_clusters.getCluster(clusterName);
+     String outputMsg = "";
+ 
+     Config rangerEnv = cluster.getDesiredConfigByType(RANGER_ENV_CONFIG_TYPE);
+ 
+     if (null == rangerEnv) {
+       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+         MessageFormat.format("Config source type {0} not found, skipping adding properties
to {1}.", RANGER_ENV_CONFIG_TYPE, RANGER_KMS_SITE_CONFIG_TYPE), "");
+     }
+ 
+     String rangerUserProp = "ranger_user";
+     String rangerUser = rangerEnv.getProperties().get(rangerUserProp);
+ 
+     if (null == rangerUser) {
+       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+         MessageFormat.format("Required user service user value from {0}/{1} not found, skipping
adding properties to {2}.", RANGER_ENV_CONFIG_TYPE, rangerUserProp, RANGER_KMS_SITE_CONFIG_TYPE),
"");
+     }
+ 
+     Config kmsSite = cluster.getDesiredConfigByType(RANGER_KMS_SITE_CONFIG_TYPE);
+ 
+     if (null == kmsSite) {
+       return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+         MessageFormat.format("Config type {0} not found, skipping adding properties to it.",
RANGER_KMS_SITE_CONFIG_TYPE), "");
+     }
+ 
+     Map<String, String> targetValues = kmsSite.getProperties();
+     if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+       String userProp = "hadoop.kms.proxyuser." + rangerUser + ".users";
+       String groupProp = "hadoop.kms.proxyuser." + rangerUser + ".groups";
+       String hostProp = "hadoop.kms.proxyuser." + rangerUser + ".hosts";
+       targetValues.put(userProp, "*");
+       targetValues.put(groupProp, "*");
+       targetValues.put(hostProp, "*");
+       kmsSite.setProperties(targetValues);
 -      kmsSite.persist(false);
++      kmsSite.save();
+       outputMsg = outputMsg + MessageFormat.format("Successfully added properties to {0}",
RANGER_KMS_SITE_CONFIG_TYPE);
+     } else {
+       outputMsg = outputMsg +  MessageFormat.format("Kerberos not enable, not setting proxy
properties to {0}", RANGER_KMS_SITE_CONFIG_TYPE);
+     }
+ 
+     return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outputMsg, "");
+ 
+   }
+ }

http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/276d1244/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
----------------------------------------------------------------------
diff --cc ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
index 0000000,e000c65..7a0d66f
mode 000000,100644..100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/RangerKmsProxyConfigTest.java
@@@ -1,0 -1,141 +1,125 @@@
+ /**
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.ambari.server.serveraction.upgrades;
+ 
+ import static org.easymock.EasyMock.anyObject;
+ import static org.easymock.EasyMock.expect;
+ import static org.easymock.EasyMock.replay;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertTrue;
+ 
+ import java.lang.reflect.Field;
+ import java.util.HashMap;
+ import java.util.Map;
+ 
+ import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+ import org.apache.ambari.server.actionmanager.HostRoleCommand;
+ import org.apache.ambari.server.agent.CommandReport;
+ import org.apache.ambari.server.agent.ExecutionCommand;
+ import org.apache.ambari.server.state.Cluster;
+ import org.apache.ambari.server.state.Clusters;
 -import org.apache.ambari.server.state.SecurityType;
+ import org.apache.ambari.server.state.Config;
 -import org.apache.ambari.server.state.ConfigImpl;
++import org.apache.ambari.server.state.SecurityType;
+ import org.easymock.EasyMock;
+ import org.junit.Before;
+ import org.junit.Test;
+ 
+ import com.google.inject.Injector;
+ 
+ 
+ public class RangerKmsProxyConfigTest {
+   private Injector m_injector;
+   private Clusters m_clusters;
+   private Field m_clusterField;
+ 
+   @Before
+   public void setup() throws Exception {
+     m_injector = EasyMock.createMock(Injector.class);
+     m_clusters = EasyMock.createMock(Clusters.class);
+     Cluster cluster = EasyMock.createMock(Cluster.class);
+ 
 -    Config rangerEnv = new ConfigImpl("ranger-env") {
 -      Map<String, String> mockProperties = new HashMap<String, String>() {{
++    Map<String, String> mockProperties = new HashMap<String, String>() {
++      {
+         put("ranger_user", "ranger");
 -      }};
 -
 -      @Override
 -      public Map<String, String> getProperties() {
 -        return mockProperties;
+       }
+     };
+ 
 -    Config kmsSite = new ConfigImpl("kms-site") {
 -      Map<String, String> mockProperties = new HashMap<String, String>();
 -      @Override
 -      public Map<String, String> getProperties() {
 -        return mockProperties;
 -      }
 -
 -      @Override
 -      public void setProperties(Map<String, String> properties) {
 -        mockProperties.putAll(properties);
 -      }
++    Config rangerEnv = EasyMock.createNiceMock(Config.class);
++    expect(rangerEnv.getType()).andReturn("ranger-env").anyTimes();
++    expect(rangerEnv.getProperties()).andReturn(mockProperties).anyTimes();
+ 
 -      @Override
 -      public void persist(boolean newConfig) {
 -        // no-op
 -      }
 -    };
++    Config kmsSite = EasyMock.createNiceMock(Config.class);
++    expect(kmsSite.getType()).andReturn("kms-site").anyTimes();
++    expect(kmsSite.getProperties()).andReturn(mockProperties).anyTimes();
+ 
+     expect(cluster.getDesiredConfigByType("ranger-env")).andReturn(rangerEnv).atLeastOnce();
+     expect(cluster.getDesiredConfigByType("kms-site")).andReturn(kmsSite).atLeastOnce();
+     expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+     expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce();
+     expect(cluster.getSecurityType()).andReturn(SecurityType.KERBEROS).anyTimes();
+ 
 -    replay(m_injector, m_clusters, cluster);
++    replay(m_injector, m_clusters, cluster, rangerEnv, kmsSite);
+ 
+     m_clusterField = RangerKmsProxyConfig.class.getDeclaredField("m_clusters");
+     m_clusterField.setAccessible(true);
+   }
+ 
+   @Test
+   public void testAction() throws Exception {
+ 
+     Map<String, String> commandParams = new HashMap<String, String>();
+     commandParams.put("clusterName", "c1");
+ 
+     ExecutionCommand executionCommand = new ExecutionCommand();
+     executionCommand.setCommandParams(commandParams);
+     executionCommand.setClusterName("c1");
+ 
+     HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+     expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+     expect(hrc.getStageId()).andReturn(2L).anyTimes();
+     expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+     replay(hrc);
+ 
+     RangerKmsProxyConfig action = new RangerKmsProxyConfig();
+     m_clusterField.set(action, m_clusters);
+ 
+     action.setExecutionCommand(executionCommand);
+     action.setHostRoleCommand(hrc);
+ 
+     CommandReport report = action.execute(null);
+     assertNotNull(report);
+ 
+     Cluster c = m_clusters.getCluster("c1");
+     Config config = c.getDesiredConfigByType("kms-site");
+     Map<String, String> map = config.getProperties();
+ 
+     assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.users"));
+     assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.groups"));
+     assertTrue(map.containsKey("hadoop.kms.proxyuser.ranger.hosts"));
+ 
+ 
+     assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.users"));
+     assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.groups"));
+     assertEquals("*", map.get("hadoop.kms.proxyuser.ranger.hosts"));
+ 
+     report = action.execute(null);
+     assertNotNull(report);
+ 
+   }
+ }


Mime
View raw message