ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mradhakrish...@apache.org
Subject [24/51] [abbrv] ambari git commit: AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement. (mpapirkovskyy)
Date Tue, 05 Dec 2017 20:35:50 GMT
AMBARI-22484. Stack advisor should disallow lzo enable without accepting license agreement.
(mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/41853a10
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/41853a10
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/41853a10

Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 41853a1041a848b69384592c49f8f5b539a5c4f9
Parents: 7750398
Author: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Authored: Wed Nov 29 16:48:11 2017 +0200
Committer: Myroslav Papirkovskyi <mpapyrkovskyy@hortonworks.com>
Committed: Wed Nov 29 18:30:08 2017 +0200

----------------------------------------------------------------------
 .../stackadvisor/StackAdvisorRequest.java       | 11 +++++
 .../commands/StackAdvisorCommand.java           |  2 +
 .../ambari/server/controller/AmbariServer.java  |  3 +-
 .../internal/StackAdvisorResourceProvider.java  |  9 +++-
 .../GPLLicenseNotAcceptedException.java         | 28 +++++++++++
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 17 +++++++
 .../ValidationResourceProviderTest.java         |  4 +-
 .../stacks/2.0.6/common/test_stack_advisor.py   | 51 ++++++++++++++++++--
 8 files changed, 116 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index cd26c56..62b8d15 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -51,6 +51,7 @@ public class StackAdvisorRequest {
   private Set<RecommendationResponse.ConfigGroup> configGroups;
   private Map<String, String> userContext = new HashMap<>();
   private Map<String, Object> ldapConfig = new HashMap<>();
+  private Boolean gplLicenseAccepted;
 
   public String getStackName() {
     return stackName;
@@ -122,6 +123,10 @@ public class StackAdvisorRequest {
     this.configGroups = configGroups;
   }
 
+  public Boolean getGplLicenseAccepted() {
+    return gplLicenseAccepted;
+  }
+
   private StackAdvisorRequest(String stackName, String stackVersion) {
     this.stackName = stackName;
     this.stackVersion = stackVersion;
@@ -194,6 +199,12 @@ public class StackAdvisorRequest {
       return this;
     }
 
+    public StackAdvisorRequestBuilder withGPLLicenseAccepted(
+        Boolean gplLicenseAccepted) {
+      this.instance.gplLicenseAccepted = gplLicenseAccepted;
+      return this;
+    }
+
     public StackAdvisorRequestBuilder withLdapConfig(Map<String, Object> ldapConfig)
{
       Preconditions.checkNotNull(ldapConfig);
       this.instance.ldapConfig = ldapConfig;

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 1b89c4f..4ec15ef 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -107,6 +107,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse>
extend
   private static final String CONFIGURATIONS_PROPERTY = "configurations";
   private static final String CHANGED_CONFIGURATIONS_PROPERTY = "changed-configurations";
   private static final String USER_CONTEXT_PROPERTY = "user-context";
+  private static final String GPL_LICENSE_ACCEPTED = "gpl-license-accepted";
   private static final String AMBARI_SERVER_CONFIGURATIONS_PROPERTY = "ambari-server-properties";
 
   private File recommendationsDir;
@@ -269,6 +270,7 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse>
extend
 
     JsonNode userContext = mapper.valueToTree(request.getUserContext());
     root.put(USER_CONTEXT_PROPERTY, userContext);
+    root.put(GPL_LICENSE_ACCEPTED, request.getGplLicenseAccepted());
   }
 
   private void populateConfigGroups(ObjectNode root,

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 6ceed4a..863313b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -908,7 +908,8 @@ public class AmbariServer {
     PersistKeyValueService.init(injector.getInstance(PersistKeyValueImpl.class));
     KeyService.init(injector.getInstance(PersistKeyValueImpl.class));
     BootStrapResource.init(injector.getInstance(BootStrapImpl.class));
-    StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class));
+    StackAdvisorResourceProvider.init(injector.getInstance(StackAdvisorHelper.class),
+        injector.getInstance(Configuration.class));
     StageUtils.setGson(injector.getInstance(Gson.class));
     StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class));
     StageUtils.setConfiguration(injector.getInstance(Configuration.class));

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 1ea664c..67c177e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -36,6 +36,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestBuilder;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.Resource.Type;
@@ -82,12 +83,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
   private static final String CONFIG_GROUPS_HOSTS_PROPERTY = "hosts";
 
   protected static StackAdvisorHelper saHelper;
+  protected static Configuration configuration;
   protected static final String USER_CONTEXT_OPERATION_PROPERTY = "user_context/operation";
   protected static final String USER_CONTEXT_OPERATION_DETAILS_PROPERTY = "user_context/operation_details";
 
   @Inject
-  public static void init(StackAdvisorHelper instance) {
+  public static void init(StackAdvisorHelper instance, Configuration serverConfig) {
     saHelper = instance;
+    configuration = serverConfig;
   }
 
   protected StackAdvisorResourceProvider(Set<String> propertyIds, Map<Type, String>
keyPropertyIds,
@@ -133,6 +136,7 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
           hgHostsMap);
       Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
       Map<String, String> userContext = readUserContext(request);
+      Boolean gplLicenseAccepted = configuration.getGplLicenseAccepted();
 
       List<ChangedConfigInfo> changedConfigurations =
         requestType == StackAdvisorRequestType.CONFIGURATION_DEPENDENCIES ?
@@ -147,7 +151,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
         withConfigurations(configurations).
         withConfigGroups(configGroups).
         withChangedConfigurations(changedConfigurations).
-        withUserContext(userContext).build();
+        withUserContext(userContext).
+        withGPLLicenseAccepted(gplLicenseAccepted).build();
     } catch (Exception e) {
       LOG.warn("Error occurred during preparation of stack advisor request", e);
       Response response = Response.status(Status.BAD_REQUEST)

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
new file mode 100644
index 0000000..b444d01
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.topology;
+
+/**
+ * Indicates an not permitted LZO usage.
+ */
+public class GPLLicenseNotAcceptedException extends Exception {
+  public GPLLicenseNotAcceptedException(String s) {
+    super(s);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index a194332..bfa2f5a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -351,6 +351,22 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     # recommendations for "hadoop.proxyuser.*.hosts", "hadoop.proxyuser.*.groups" properties
in core-site
     self.recommendHadoopProxyUsers(configurations, services, hosts)
 
+  def getLZOSupportValidationItems(self, properties, services):
+    services_list = self.get_services_list(services)
+
+    if "HDFS" in services_list:
+      lzo_allowed = services["gpl-license-accepted"]
+      property_name = "io.compression.codec.lzo.class"
+      if property_name in properties:
+        property_value = properties.get(property_name)
+        if not lzo_allowed and "com.hadoop.compression.lzo.LzoCodec" in property_value:
+          return [{"config-name": property_name, "item": self.getErrorItem(
+            "Your Ambari Server has not been configured to download LZO and install it. "
+            "LZO is GPL software and requires you to accept a license prior to use. "
+            "Please refer to this documentation to configure Ambari before proceeding.")}]
+
+    return []
+
   def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
     # recommendations for HBase env config
 
@@ -726,6 +742,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     validationItems = []
     validationItems.extend(self.getHadoopProxyUsersValidationItems(properties, services,
hosts, configurations))
     validationItems.extend(self.getAmbariProxyUsersForHDFSValidationItems(properties, services))
+    validationItems.extend(self.getLZOSupportValidationItems(properties, services))
     return self.toConfigurationValidationProblems(validationItems, "core-site")
 
   def validatorOneDataDirPerPartition(self, properties, propertyName, services, hosts, clusterEnv):

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
index c639d1f..42d4770 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ValidationResourceProviderTest.java
@@ -33,6 +33,7 @@ import java.util.Set;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
 import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.Request;
 import org.apache.ambari.server.controller.spi.RequestStatus;
@@ -53,13 +54,14 @@ public class ValidationResourceProviderTest {
     doReturn(stackAdvisorRequest).when(provider).prepareStackAdvisorRequest(request);
 
     StackAdvisorHelper saHelper = mock(StackAdvisorHelper.class);
+    Configuration configuration = mock(Configuration.class);
 
     ValidationResponse response = mock(ValidationResponse.class);
     Version version = mock(Version.class);
     doReturn(3).when(response).getId();
     doReturn(version).when(response).getVersion();
     doReturn(response).when(saHelper).validate(any(StackAdvisorRequest.class));
-    ValidationResourceProvider.init(saHelper);
+    ValidationResourceProvider.init(saHelper, configuration);
 
     RequestStatus status = provider.createResources(request);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/41853a10/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index 38d6ecd..6c774af 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -2521,7 +2521,8 @@ class TestHDP206StackAdvisor(TestCase):
                   'hadoop.proxyuser.hdfs-user.hosts': '*',
                   'hadoop.proxyuser.hdfs-user.groups': '*',
                   'hadoop.proxyuser.yarn-user.hosts': 'host1,host2',
-                  'hadoop.proxyuser.yarn-user.groups': '*'}
+                  'hadoop.proxyuser.yarn-user.groups': '*',
+                  'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
     services = {
       'services':  [
         { 'StackServices': {'service_name': 'HDFS'}},
@@ -2537,7 +2538,8 @@ class TestHDP206StackAdvisor(TestCase):
         }
       ],
       'ambari-server-properties': {'ambari-server.user': 'ambari-user'},
-      'configurations': configurations
+      'configurations': configurations,
+      "gpl-license-accepted": True
     }
     hosts = {
       'items' : [
@@ -2551,7 +2553,21 @@ class TestHDP206StackAdvisor(TestCase):
     res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults,
configurations, services, hosts)
     self.assertEquals(res, res_expected)
 
-    # 2) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts
is expected to be set
+    # 2) fail: gpl is not allowed
+    services["gpl-license-accepted"] = False
+    res_expected = [{'config-type': 'core-site',
+                     'message': 'Your Ambari Server has not been configured to download LZO
and install it. '
+                                'LZO is GPL software and requires you to accept a license
prior to use. '
+                                'Please refer to this documentation to configure Ambari before
proceeding.',
+                     'type': 'configuration',
+                     'config-name': 'io.compression.codec.lzo.class',
+                     'level': 'ERROR'}]
+
+    res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations,
services, hosts)
+    self.assertEquals(res, res_expected)
+    services["gpl-license-accepted"] = True
+
+    # 3) fail: test filter function: two RESOURCE_MANAGERs, hadoop.proxyuser.yarn-user.hosts
is expected to be set
     del properties['hadoop.proxyuser.yarn-user.hosts']
     res_expected = [{'config-name': 'hadoop.proxyuser.yarn-user.hosts',
                      'config-type': 'core-site',
@@ -2561,13 +2577,13 @@ class TestHDP206StackAdvisor(TestCase):
     res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults,
configurations, services, hosts)
     self.assertEquals(res, res_expected)
 
-    # 3) ok: test filter function: only one RESOURCE_MANAGER
+    # 4) ok: test filter function: only one RESOURCE_MANAGER
     services['services'][1]['components'][0]['StackServiceComponents']['hostnames'] = ["host1"]
     res_expected = []
     res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, recommendedDefaults,
configurations, services, hosts)
     self.assertEquals(res, res_expected)
 
-    # 4) fail: some proxyusers are empty or absent:
+    # 5) fail: some proxyusers are empty or absent:
     del properties['hadoop.proxyuser.ambari-user.hosts']
     properties['hadoop.proxyuser.hdfs-user.groups'] = ''
     res_expected = [{'config-name': 'hadoop.proxyuser.hdfs-user.groups',
@@ -2627,6 +2643,31 @@ class TestHDP206StackAdvisor(TestCase):
     res = self.stackAdvisor.getHadoopProxyUsers(services, hosts, configurations)
     self.assertEquals(res, res_expected)
 
+  # def test_validateHDFSConfigurationsCoreSite(self):
+  #
+  #   configurations = {}
+  #   services = {"gpl-license-accepted": True, "services": [{"StackServices": {"service_name":
"HDFS"}}], 'ambari-server-properties': {'ambari-server.user': 'ambari-user'}}
+  #
+  #   # 1) ok: gpl is allowed
+  #   properties = {'io.compression.codec.lzo.class': 'com.hadoop.compression.lzo.LzoCodec'}
+  #   res_expected = []
+  #
+  #   res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations,
services, '')
+  #   self.assertEquals(res, res_expected)
+  #
+  #   # 2) fail: gpl is not allowed
+  #   services["gpl-license-accepted"] = False
+  #   res_expected = [{'config-type': 'core-site',
+  #                    'message': 'Your Ambari Server has not been configured to download
LZO and install it. '
+  #                               'LZO is GPL software and requires you to accept a license
prior to use. '
+  #                               'Please refer to this documentation to configure Ambari
before proceeding.',
+  #                    'type': 'configuration',
+  #                    'config-name': 'io.compression.codec.lzo.class',
+  #                    'level': 'ERROR'}]
+  #
+  #   res = self.stackAdvisor.validateHDFSConfigurationsCoreSite(properties, {}, configurations,
services, '')
+  #   self.assertEquals(res, res_expected)
+
   def test_validateOneDataDirPerPartition(self):
     recommendedDefaults = {
       'dfs.datanode.du.reserved': '1024'


Mime
View raw message