Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 55BFD200CB2 for ; Sun, 25 Jun 2017 11:59:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 549B3160BD8; Sun, 25 Jun 2017 09:59:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 108D5160C06 for ; Sun, 25 Jun 2017 11:59:41 +0200 (CEST) Received: (qmail 53706 invoked by uid 500); 25 Jun 2017 09:59:40 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 52666 invoked by uid 99); 25 Jun 2017 09:59:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 25 Jun 2017 09:59:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC780E96C0; Sun, 25 Jun 2017 09:59:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hapylestat@apache.org To: commits@ambari.apache.org Date: Sun, 25 Jun 2017 09:59:56 -0000 Message-Id: <0354b573f124449299041e7779163d62@git.apache.org> In-Reply-To: <4be36fd3736f4a4c8aad1fbf40b2120d@git.apache.org> References: <4be36fd3736f4a4c8aad1fbf40b2120d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [20/25] ambari git commit: AMBARI-21268 Remove Upgrade Catalogs For Every Version Before 2.5 (dgrinenko) archived-at: Sun, 25 Jun 2017 09:59:45 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/af1bf85c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java deleted file mode 100644 index f413c69..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ /dev/null @@ -1,3079 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.upgrade; - -import java.sql.Clob; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - -import javax.persistence.EntityManager; -import javax.persistence.Query; -import javax.persistence.TypedQuery; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.agent.RecoveryConfigHelper; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; -import org.apache.ambari.server.orm.dao.AlertDefinitionDAO; -import org.apache.ambari.server.orm.dao.ArtifactDAO; -import org.apache.ambari.server.orm.dao.ClusterDAO; -import org.apache.ambari.server.orm.dao.PermissionDAO; -import org.apache.ambari.server.orm.dao.PrincipalDAO; -import org.apache.ambari.server.orm.dao.PrincipalTypeDAO; -import org.apache.ambari.server.orm.dao.PrivilegeDAO; -import org.apache.ambari.server.orm.dao.RemoteAmbariClusterDAO; -import org.apache.ambari.server.orm.dao.RequestScheduleDAO; -import org.apache.ambari.server.orm.dao.ResourceTypeDAO; -import org.apache.ambari.server.orm.dao.RoleAuthorizationDAO; -import org.apache.ambari.server.orm.dao.UserDAO; -import org.apache.ambari.server.orm.dao.ViewInstanceDAO; -import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; -import org.apache.ambari.server.orm.entities.ArtifactEntity; -import org.apache.ambari.server.orm.entities.ClusterEntity; -import org.apache.ambari.server.orm.entities.PermissionEntity; -import org.apache.ambari.server.orm.entities.PrincipalEntity; -import org.apache.ambari.server.orm.entities.PrincipalTypeEntity; -import org.apache.ambari.server.orm.entities.PrivilegeEntity; -import org.apache.ambari.server.orm.entities.RemoteAmbariClusterEntity; -import org.apache.ambari.server.orm.entities.RequestScheduleEntity; -import org.apache.ambari.server.orm.entities.ResourceEntity; -import org.apache.ambari.server.orm.entities.ResourceTypeEntity; -import org.apache.ambari.server.orm.entities.RoleAuthorizationEntity; -import org.apache.ambari.server.orm.entities.UserEntity; -import org.apache.ambari.server.orm.entities.ViewEntityEntity; -import org.apache.ambari.server.orm.entities.ViewInstanceEntity; -import org.apache.ambari.server.security.authorization.ResourceType; -import org.apache.ambari.server.security.authorization.User; -import org.apache.ambari.server.security.authorization.Users; -import org.apache.ambari.server.state.AlertFirmness; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.state.RepositoryType; -import org.apache.ambari.server.state.SecurityType; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceInfo; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.StackInfo; -import org.apache.ambari.server.state.State; -import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; -import org.apache.ambari.server.state.kerberos.KerberosDescriptor; -import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory; -import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor; -import org.apache.ambari.server.state.kerberos.KerberosKeytabDescriptor; -import org.apache.ambari.server.state.kerberos.KerberosPrincipalDescriptor; -import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; -import org.apache.ambari.server.view.DefaultMasker; -import org.apache.ambari.view.ClusterType; -import org.apache.ambari.view.MaskException; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.jdbc.support.JdbcUtils; - -import com.google.common.collect.Lists; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonPrimitive; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.persist.Transactional; - -/** - * Upgrade catalog for version 2.4.0. - */ -public class UpgradeCatalog240 extends AbstractUpgradeCatalog { - - protected static final String ADMIN_PERMISSION_TABLE = "adminpermission"; - protected static final String PRINCIPAL_ID_COL = "principal_id"; - protected static final String ALERT_DEFINITION_TABLE = "alert_definition"; - protected static final String ALERT_TARGET_TABLE = "alert_target"; - protected static final String ALERT_TARGET_ENABLED_COLUMN = "is_enabled"; - protected static final String ALERT_CURRENT_TABLE = "alert_current"; - protected static final String ALERT_CURRENT_OCCURRENCES_COLUMN = "occurrences"; - protected static final String ALERT_CURRENT_FIRMNESS_COLUMN = "firmness"; - protected static final String HELP_URL_COLUMN = "help_url"; - protected static final String REPEAT_TOLERANCE_COLUMN = "repeat_tolerance"; - protected static final String REPEAT_TOLERANCE_ENABLED_COLUMN = "repeat_tolerance_enabled"; - protected static final String PERMISSION_ID_COL = "permission_name"; - protected static final String SORT_ORDER_COL = "sort_order"; - protected static final String REPO_VERSION_TABLE = "repo_version"; - protected static final String HOST_ROLE_COMMAND_TABLE = "host_role_command"; - protected static final String SERVICE_COMPONENT_DS_TABLE = "servicecomponentdesiredstate"; - protected static final String HOST_COMPONENT_DS_TABLE = "hostcomponentdesiredstate"; - protected static final String HOST_COMPONENT_STATE_TABLE = "hostcomponentstate"; - protected static final String SERVICE_COMPONENT_HISTORY_TABLE = "servicecomponent_history"; - protected static final String UPGRADE_TABLE = "upgrade"; - protected static final String STACK_TABLE = "stack"; - protected static final String CLUSTER_TABLE = "clusters"; - protected static final String CLUSTER_UPGRADE_ID_COLUMN = "upgrade_id"; - protected static final String YARN_ENV_CONFIG = "yarn-env"; - protected static final String CAPACITY_SCHEDULER_CONFIG = "capacity-scheduler"; - protected static final String WEBHCAT_SITE_CONFIG = "webhcat-site"; - protected static final String TEZ_SITE_CONFIG = "tez-site"; - protected static final String MAPRED_SITE_CONFIG = "mapred-site"; - public static final String DESIRED_VERSION_COLUMN_NAME = "desired_version"; - public static final String BLUEPRINT_SETTING_TABLE = "blueprint_setting"; - public static final String BLUEPRINT_NAME_COL = "blueprint_name"; - public static final String SETTING_NAME_COL = "setting_name"; - public static final String SETTING_DATA_COL = "setting_data"; - public static final String ID = "id"; - public static final String BLUEPRINT_TABLE = "blueprint"; - public static final String VIEWINSTANCE_TABLE = "viewinstance"; - public static final String SHORT_URL_COLUMN = "short_url"; - public static final String CLUSTER_HANDLE_COLUMN = "cluster_handle"; - public static final String REQUESTSCHEDULE_TABLE = "requestschedule"; - public static final String AUTHENTICATED_USER_ID_COLUMN = "authenticated_user_id"; - protected static final String CLUSTER_VERSION_TABLE = "cluster_version"; - protected static final String HOST_VERSION_TABLE = "host_version"; - protected static final String TOPOLOGY_REQUEST_TABLE = "topology_request"; - protected static final String PROVISION_ACTION_COL = "provision_action"; - protected static final String PHOENIX_QUERY_SERVER_PRINCIPAL_KEY = "phoenix.queryserver.kerberos.principal"; - protected static final String PHOENIX_QUERY_SERVER_KEYTAB_KEY = "phoenix.queryserver.keytab.file"; - protected static final String DEFAULT_CONFIG_VERSION = "version1"; - protected static final String SLIDER_SERVICE_NAME = "SLIDER"; - - private static final String OOZIE_ENV_CONFIG = "oozie-env"; - private static final String SLIDER_CLIENT_CONFIG = "slider-client"; - private static final String HIVE_ENV_CONFIG = "hive-env"; - private static final String AMS_SITE = "ams-site"; - public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD = "timeline.metrics.sink.collection.period"; - public static final String ONE_DIR_PER_PARITION_PROPERTY = "one_dir_per_partition"; - public static final String VIEWURL_TABLE = "viewurl"; - public static final String URL_ID_COLUMN = "url_id"; - private static final String PRINCIPAL_TYPE_TABLE = "adminprincipaltype"; - private static final String PRINCIPAL_TABLE = "adminprincipal"; - protected static final String HBASE_SITE_CONFIG = "hbase-site"; - protected static final String HBASE_SPNEGO_PRINCIPAL_KEY = "hbase.security.authentication.spnego.kerberos.principal"; - protected static final String HBASE_SPNEGO_KEYTAB_KEY = "hbase.security.authentication.spnego.kerberos.keytab"; - protected static final String EXTENSION_TABLE = "extension"; - protected static final String EXTENSION_ID_COLUMN = "extension_id"; - protected static final String EXTENSION_LINK_TABLE = "extensionlink"; - protected static final String EXTENSION_LINK_ID_COLUMN = "link_id"; - protected static final String KAFKA_BROKER_CONFIG = "kafka-broker"; - - private static final Map ROLE_ORDER; - private static final String AMS_HBASE_SITE = "ams-hbase-site"; - private static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout"; - private static final String AMS_HBASE_SITE_NORMALIZER_ENABLED_PROPERTY = "hbase.normalizer.enabled"; - public static final String PRECISION_TABLE_TTL_PROPERTY = "timeline.metrics.host.aggregator.ttl"; - public static final String CLUSTER_SECOND_TABLE_TTL_PROPERTY = "timeline.metrics.cluster.aggregator.second.ttl"; - - static { - // Manually create role order since there really isn't any mechanism for this - ROLE_ORDER = new HashMap<>(); - ROLE_ORDER.put("AMBARI.ADMINISTRATOR", 1); - ROLE_ORDER.put("CLUSTER.ADMINISTRATOR", 2); - ROLE_ORDER.put("CLUSTER.OPERATOR", 3); - ROLE_ORDER.put("SERVICE.ADMINISTRATOR", 4); - ROLE_ORDER.put("SERVICE.OPERATOR", 5); - ROLE_ORDER.put("CLUSTER.USER", 6); - } - - @Inject - UserDAO userDAO; - - @Inject - PermissionDAO permissionDAO; - - @Inject - PrivilegeDAO privilegeDAO; - - @Inject - ResourceTypeDAO resourceTypeDAO; - - @Inject - ClusterDAO clusterDAO; - - @Inject - PrincipalTypeDAO principalTypeDAO; - - @Inject - PrincipalDAO principalDAO; - - @Inject - RequestScheduleDAO requestScheduleDAO; - - @Inject - Users users; - - @Inject - Configuration config; - - /** - * Logger. - */ - private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog240.class); - - private static final String SETTING_TABLE = "setting"; - - protected static final String SERVICE_COMPONENT_DESIRED_STATE_TABLE = "servicecomponentdesiredstate"; - protected static final String RECOVERY_ENABLED_COL = "recovery_enabled"; - - // ----- Constructors ------------------------------------------------------ - - /** - * Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()} - * - * @param injector Guice injector to track dependencies and uses bindings to inject them. - */ - @Inject - public UpgradeCatalog240(Injector injector) { - super(injector); - injector.injectMembers(this); - } - - // ----- UpgradeCatalog ---------------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getTargetVersion() { - return "2.4.0"; - } - - // ----- AbstractUpgradeCatalog -------------------------------------------- - - /** - * {@inheritDoc} - */ - @Override - public String getSourceVersion() { - return "2.3.0"; - } - - public static final String CLUSTER_TYPE_COLUMN = "cluster_type"; - public static final String REMOTE_AMBARI_CLUSTER_TABLE = "remoteambaricluster"; - public static final String REMOTE_AMBARI_CLUSTER_SERVICE_TABLE = "remoteambariclusterservice"; - - public static final String CLUSTER_ID = "cluster_id"; - public static final String SERVICE_NAME = "service_name"; - public static final String CLUSTER_NAME = "name"; - - - @Override - protected void executeDDLUpdates() throws AmbariException, SQLException { - updateAdminPermissionTable(); - updateServiceComponentDesiredStateTable(); - createExtensionTable(); - createExtensionLinkTable(); - createSettingTable(); - updateRepoVersionTableDDL(); - updateServiceComponentDesiredStateTableDDL(); - createServiceComponentHistoryTable(); - updateClusterTableDDL(); - updateAlertDefinitionTable(); - updateAlertCurrentTable(); - updateAlertTargetTable(); - createBlueprintSettingTable(); - updateHostRoleCommandTableDDL(); - createViewUrlTableDDL(); - updateViewInstanceEntityTable(); - createRemoteClusterTable(); - updateViewInstanceTable(); - updateRequestScheduleEntityTable(); - updateTopologyRequestTable(); - } - - private void createRemoteClusterTable() throws SQLException { - - List columns = new ArrayList<>(); - LOG.info("Creating {} table", REMOTE_AMBARI_CLUSTER_TABLE); - columns.add(new DBColumnInfo(CLUSTER_ID, Long.class, null, null, false)); - columns.add(new DBColumnInfo(CLUSTER_NAME, String.class, 255, null, false)); - columns.add(new DBColumnInfo("url", String.class, 255, null, false)); - columns.add(new DBColumnInfo("username", String.class, 255, null, false)); - columns.add(new DBColumnInfo("password", String.class, 255, null, false)); - dbAccessor.createTable(REMOTE_AMBARI_CLUSTER_TABLE, columns, CLUSTER_ID); - dbAccessor.addUniqueConstraint(REMOTE_AMBARI_CLUSTER_TABLE , "UQ_remote_ambari_cluster" , CLUSTER_NAME); - addSequence("remote_cluster_id_seq", 1L, false); - - List remoteClusterServiceColumns = new ArrayList<>(); - LOG.info("Creating {} table", REMOTE_AMBARI_CLUSTER_SERVICE_TABLE); - remoteClusterServiceColumns.add(new DBColumnInfo(ID, Long.class, null, null, false)); - remoteClusterServiceColumns.add(new DBColumnInfo(SERVICE_NAME, String.class, 255, null, false)); - remoteClusterServiceColumns.add(new DBColumnInfo(CLUSTER_ID, Long.class, null, null, false)); - dbAccessor.createTable(REMOTE_AMBARI_CLUSTER_SERVICE_TABLE, remoteClusterServiceColumns, ID); - dbAccessor.addFKConstraint(REMOTE_AMBARI_CLUSTER_SERVICE_TABLE, "FK_remote_ambari_cluster_id", - CLUSTER_ID, REMOTE_AMBARI_CLUSTER_TABLE, CLUSTER_ID, false); - addSequence("remote_cluster_service_id_seq", 1L, false); - - } - - private void createViewUrlTableDDL() throws SQLException { - List columns = new ArrayList<>(); - - // Add setting table - LOG.info("Creating " + VIEWURL_TABLE + " table"); - - columns.add(new DBColumnInfo(URL_ID_COLUMN, Long.class, null, null, false)); - columns.add(new DBColumnInfo("url_name", String.class, 255, null, false)); - columns.add(new DBColumnInfo("url_suffix", String.class, 255, null, false)); - dbAccessor.createTable(VIEWURL_TABLE, columns, URL_ID_COLUMN); - addSequence("viewurl_id_seq", 1L, false); - } - - private void updateViewInstanceEntityTable() throws SQLException { - dbAccessor.addColumn(VIEWINSTANCE_TABLE, - new DBColumnInfo(SHORT_URL_COLUMN, Long.class, null, null, true)); - dbAccessor.addFKConstraint(VIEWINSTANCE_TABLE, "FK_instance_url_id", - SHORT_URL_COLUMN, VIEWURL_TABLE, URL_ID_COLUMN, false); - dbAccessor.addColumn(VIEWINSTANCE_TABLE, - new DBColumnInfo(CLUSTER_TYPE_COLUMN, String.class, 100, ClusterType.LOCAL_AMBARI.name(), false)); - } - - private void updateRequestScheduleEntityTable() throws SQLException { - dbAccessor.addColumn(REQUESTSCHEDULE_TABLE, - new DBColumnInfo(AUTHENTICATED_USER_ID_COLUMN, Integer.class, null, null, true)); - } - - private void updateClusterTableDDL() throws SQLException { - dbAccessor.addColumn(CLUSTER_TABLE, new DBColumnInfo(CLUSTER_UPGRADE_ID_COLUMN, Long.class, null, null, true)); - - dbAccessor.addFKConstraint(CLUSTER_TABLE, "FK_clusters_upgrade_id", - CLUSTER_UPGRADE_ID_COLUMN, UPGRADE_TABLE, "upgrade_id", false); - } - - @Override - protected void executePreDMLUpdates() throws AmbariException, SQLException { - //To change body of implemented methods use File | Settings | File Templates. - } - - @Override - protected void executeDMLUpdates() throws AmbariException, SQLException { - addNewConfigurationsFromXml(); - updateAlerts(); - setRoleSortOrder(); - addSettingPermission(); - addViewOperationalLogsPermission(); - addManageUserPersistedDataPermission(); - allowClusterOperatorToManageCredentials(); - updateHDFSConfigs(); - updateKAFKAConfigs(); - updateHIVEConfigs(); - updateAMSConfigs(); - updateClusterEnv(); - updateSequenceForView(); - updateHostRoleCommandTableDML(); - updateKerberosConfigs(); - updateYarnEnv(); - updatePhoenixConfigs(); - updateSparkConfigs(); - updateHBaseConfigs(); - updateFalconConfigs(); - updateKerberosDescriptorArtifacts(); - removeHiveOozieDBConnectionConfigs(); - updateClustersAndHostsVersionStateTableDML(); - removeStandardDeviationAlerts(); - removeAtlasMetaserverAlert(); - updateClusterInheritedPermissionsConfig(); - consolidateUserRoles(); - createRolePrincipals(); - updateHDFSWidgetDefinition(); - updateTezViewProperty(); - upgradeCapSchedulerView(); - fixAuthorizationDescriptions(); - removeAuthorizations(); - addConnectionTimeoutParamForWebAndMetricAlerts(); - addSliderClientConfig(); - updateRequestScheduleEntityUserIds(); - updateRecoveryConfigurationDML(); - updatePigSmokeTestEntityClass(); - updateRangerHbasePluginProperties(); - adjustHiveJobTimestamps(); - } - - /** - * Populates authenticated_user_id field by correct user id calculated from user name - * @throws SQLException - */ - protected void updateRequestScheduleEntityUserIds() throws SQLException { - List requestScheduleEntities = requestScheduleDAO.findAll(); - for (RequestScheduleEntity requestScheduleEntity : requestScheduleEntities) { - String createdUserName = requestScheduleEntity.getCreateUser(); - - if (createdUserName != null) { - User user = users.getUserIfUnique(createdUserName); - - if (user != null && StringUtils.equals(user.getUserName(), createdUserName)) { - requestScheduleEntity.setAuthenticatedUserId(user.getUserId()); - requestScheduleDAO.merge(requestScheduleEntity); - } - } - } - } - - protected void updateClusterInheritedPermissionsConfig() throws SQLException { - insertClusterInheritedPrincipal("ALL.CLUSTER.ADMINISTRATOR"); - insertClusterInheritedPrincipal("ALL.CLUSTER.OPERATOR"); - insertClusterInheritedPrincipal("ALL.CLUSTER.USER"); - insertClusterInheritedPrincipal("ALL.SERVICE.ADMINISTRATOR"); - insertClusterInheritedPrincipal("ALL.SERVICE.OPERATIOR"); - } - - private void insertClusterInheritedPrincipal(String name) { - PrincipalTypeEntity principalTypeEntity = new PrincipalTypeEntity(); - principalTypeEntity.setName(name); - principalTypeEntity = principalTypeDAO.merge(principalTypeEntity); - - PrincipalEntity principalEntity = new PrincipalEntity(); - principalEntity.setPrincipalType(principalTypeEntity); - principalDAO.create(principalEntity); - } - private static final String NAME_PREFIX = "DS_"; - - private String getEntityName(ViewEntityEntity entity) { - String className = entity.getClassName(); - String[] parts = className.split("\\."); - String simpleClassName = parts[parts.length - 1]; - - if (entity.getViewInstance().alterNames()) { - return NAME_PREFIX + simpleClassName + "_" + entity.getId(); - } - return simpleClassName + entity.getId(); - } - - /** - * get all entries of viewentity - * find all the table names by parsing class_name - * create all the sequence names by appending _id_seq - * query each dynamic table to find the max of id - * insert into ambari_sequence name and counter for each item - */ - protected void updateSequenceForView() { - LOG.info("updateSequenceForView called."); - EntityManager entityManager = getEntityManagerProvider().get(); - TypedQuery viewEntityQuery = entityManager.createQuery("SELECT vee FROM ViewEntityEntity vee", ViewEntityEntity.class); - List viewEntities = viewEntityQuery.getResultList(); - LOG.info("Received view Entities : {}, length : {}", viewEntities, viewEntities.size()); - - // as the id fields are string in these entities we will have to get all ids and convert to int and find max. - String selectIdsFormat = "select %s from %s"; - String insertQuery = "insert into ambari_sequences values ('%s',%d)"; - for (ViewEntityEntity viewEntity : viewEntities) { - LOG.info("Working with viewEntity : {} : {} ", viewEntity, viewEntity.getViewName() + viewEntity.getViewInstance()); - String tableName = getEntityName(viewEntity); - String seqName = tableName.toLowerCase() + "_id_seq"; - try { - entityManager.getTransaction().begin(); - String selectIdsQueryString = String.format(selectIdsFormat, NAME_PREFIX + viewEntity.getIdProperty(), tableName).toLowerCase(); - LOG.info("executing max query string {}", selectIdsQueryString); - Query selectIdsQuery = entityManager.createNativeQuery(selectIdsQueryString); - List ids = selectIdsQuery.getResultList(); - LOG.info("Received ids : {}", ids); - int maxId = 0; - if (null != ids && ids.size() != 0) { - for (String id : ids) { - try { - Integer intId = Integer.parseInt(id); - maxId = Math.max(intId, maxId); - } catch (NumberFormatException e) { - LOG.error("the id was non integer : id : {}. So ignoring.", id); - } - } - } - - String insertQueryString = String.format(insertQuery, seqName, maxId).toLowerCase(); - LOG.info("Executing insert query : {}", insertQueryString); - Query insertQ = entityManager.createNativeQuery(insertQueryString); - int rowsChanged = insertQ.executeUpdate(); - entityManager.getTransaction().commit(); - LOG.info("executing insert resulted in {} row changes.", rowsChanged); - } catch (Exception e) { // when the entity table is not yet created or other exception. - entityManager.getTransaction().rollback(); - LOG.info("Error (can be ignored) {}", e.getMessage()); - LOG.debug("Exception occured while updating : {}",viewEntity.getViewName() + viewEntity.getViewInstance(), e); - } - } - } - - - /** - * get all entries of viewentity - * find all the table names by parsing class_name - * update jobimpls creation timestamp * 1000 - */ - protected void adjustHiveJobTimestamps() { - LOG.info("updateSequenceForView called."); - EntityManager entityManager = getEntityManagerProvider().get(); - TypedQuery viewEntityQuery = entityManager.createQuery("SELECT vee FROM ViewEntityEntity vee where vee.className = 'org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl'", ViewEntityEntity.class); - List viewEntities = viewEntityQuery.getResultList(); - LOG.info("Received JobImpl view Entities : {}, length : {}", viewEntities, viewEntities.size()); - - String selectIdsFormat = "update %s set ds_datesubmitted = ds_datesubmitted * 1000"; - for (ViewEntityEntity viewEntity : viewEntities) { - LOG.info("Working with JobImpl viewEntity : {} : {} ", viewEntity, viewEntity.getViewName() + ":" + viewEntity.getViewInstanceName() + ":" + viewEntity.getClassName()); - String tableName = getEntityName(viewEntity); - try { - entityManager.getTransaction().begin(); - String updatesQueryString = String.format(selectIdsFormat, tableName).toLowerCase(); - LOG.info("executing update query string for jobimpl {}", updatesQueryString); - Query updateQuery = entityManager.createNativeQuery(updatesQueryString); - int rowsChanged = updateQuery.executeUpdate(); - entityManager.getTransaction().commit(); - LOG.info("executing update on jobimpl resulted in {} row changes.", rowsChanged); - } catch (Exception e) { // when the entity table is not yet created or other exception. - entityManager.getTransaction().rollback(); - LOG.info("Error (can be ignored) {}", e.getMessage()); - LOG.debug("Exception occured while updating : {}",viewEntity.getViewName() + viewEntity.getViewInstance(), e); - } - } - } - - private void createExtensionTable() throws SQLException { - List columns = new ArrayList<>(); - - // Add extension table - LOG.info("Creating " + EXTENSION_TABLE + " table"); - - columns.add(new DBColumnInfo(EXTENSION_ID_COLUMN, Long.class, null, null, false)); - columns.add(new DBColumnInfo("extension_name", String.class, 255, null, false)); - columns.add(new DBColumnInfo("extension_version", String.class, 255, null, false)); - dbAccessor.createTable(EXTENSION_TABLE, columns, EXTENSION_ID_COLUMN); - - // create UNIQUE constraint, ensuring column order matches SQL files - String[] uniqueColumns = new String[] { "extension_name", "extension_version" }; - dbAccessor.addUniqueConstraint(EXTENSION_TABLE, "UQ_extension", uniqueColumns); - - addSequence("extension_id_seq", 0L, false); - } - - private void createExtensionLinkTable() throws SQLException { - List columns = new ArrayList<>(); - - // Add extension link table - LOG.info("Creating " + EXTENSION_LINK_TABLE + " table"); - - columns.add(new DBColumnInfo(EXTENSION_LINK_ID_COLUMN, Long.class, null, null, false)); - columns.add(new DBColumnInfo("stack_id", Long.class, null, null, false)); - columns.add(new DBColumnInfo(EXTENSION_ID_COLUMN, Long.class, null, null, false)); - dbAccessor.createTable(EXTENSION_LINK_TABLE, columns, EXTENSION_LINK_ID_COLUMN); - - // create UNIQUE constraint, ensuring column order matches SQL files - String[] uniqueColumns = new String[] { "stack_id", EXTENSION_ID_COLUMN }; - dbAccessor.addUniqueConstraint(EXTENSION_LINK_TABLE, "UQ_extension_link", uniqueColumns); - - dbAccessor.addFKConstraint(EXTENSION_LINK_TABLE, "FK_extensionlink_extension_id", - EXTENSION_ID_COLUMN, EXTENSION_TABLE, EXTENSION_ID_COLUMN, false); - - dbAccessor.addFKConstraint(EXTENSION_LINK_TABLE, "FK_extensionlink_stack_id", - "stack_id", STACK_TABLE, "stack_id", false); - - addSequence("link_id_seq", 0L, false); - } - - private void createSettingTable() throws SQLException { - List columns = new ArrayList<>(); - - // Add setting table - LOG.info("Creating " + SETTING_TABLE + " table"); - - columns.add(new DBColumnInfo(ID, Long.class, null, null, false)); - columns.add(new DBColumnInfo("name", String.class, 255, null, false)); - columns.add(new DBColumnInfo("setting_type", String.class, 255, null, false)); - columns.add(new DBColumnInfo("content", String.class, 3000, null, false)); - columns.add(new DBColumnInfo("updated_by", String.class, 255, "_db", false)); - columns.add(new DBColumnInfo("update_timestamp", Long.class, null, null, false)); - dbAccessor.createTable(SETTING_TABLE, columns, ID); - addSequence("setting_id_seq", 0L, false); - } - - protected void addSettingPermission() throws SQLException { - addRoleAuthorization("AMBARI.MANAGE_SETTINGS", "Manage settings", Collections.singleton("AMBARI.ADMINISTRATOR:AMBARI")); - } - - protected void addViewOperationalLogsPermission() throws SQLException { - Collection roles = Arrays.asList( - "AMBARI.ADMINISTRATOR:AMBARI", - "CLUSTER.ADMINISTRATOR:CLUSTER", - "CLUSTER.OPERATOR:CLUSTER", - "SERVICE.ADMINISTRATOR:CLUSTER"); - - addRoleAuthorization("SERVICE.VIEW_OPERATIONAL_LOGS", "View service operational logs", roles); - } - - /** - * Add 'MANAGE_USER_PERSISTED_DATA' permissions for CLUSTER.ADMINISTRATOR, SERVICE.OPERATOR, SERVICE.ADMINISTRATOR, - * CLUSTER.OPERATOR, AMBARI.ADMINISTRATOR. - * - */ - protected void addManageUserPersistedDataPermission() throws SQLException { - Collection roles = Arrays.asList( - "AMBARI.ADMINISTRATOR:AMBARI", - "CLUSTER.ADMINISTRATOR:CLUSTER", - "CLUSTER.OPERATOR:CLUSTER", - "SERVICE.ADMINISTRATOR:CLUSTER", - "SERVICE.OPERATOR:CLUSTER", - "CLUSTER.USER:CLUSTER"); - - addRoleAuthorization("CLUSTER.MANAGE_USER_PERSISTED_DATA", "Manage cluster-level user persisted data", roles); - } - - /** - * Adds CLUSTER.MANAGE_CREDENTIALS to the set of authorizations a CLUSTER.OPERATOR can perform. - * - * @throws SQLException - */ - protected void allowClusterOperatorToManageCredentials() throws SQLException { - addAuthorizationToRole("CLUSTER.OPERATOR", "CLUSTER", "CLUSTER.MANAGE_CREDENTIAL"); - } - - protected void removeHiveOozieDBConnectionConfigs() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Map clusterMap = getCheckedClusterMap(ambariManagementController.getClusters()); - - for (final Cluster cluster : clusterMap.values()) { - Config oozieEnv = cluster.getDesiredConfigByType(OOZIE_ENV_CONFIG); - if(oozieEnv != null) { - Map oozieEnvProperties = oozieEnv.getProperties(); - Set removePropertiesSet = new HashSet<>(); - if (oozieEnvProperties.containsKey("oozie_derby_database")) { - LOG.info("Removing property oozie_derby_database from " + OOZIE_ENV_CONFIG); - removePropertiesSet.add("oozie_derby_database"); - } - if (oozieEnvProperties.containsKey("oozie_hostname")) { - LOG.info("Removing property oozie_hostname from " + OOZIE_ENV_CONFIG); - removePropertiesSet.add("oozie_hostname"); - } - if (!removePropertiesSet.isEmpty()) { - removeConfigurationPropertiesFromCluster(cluster, OOZIE_ENV_CONFIG, removePropertiesSet); - } - } - - Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV_CONFIG); - if(hiveEnv != null) { - Map hiveEnvProperties = hiveEnv.getProperties(); - if (hiveEnvProperties.containsKey("hive_hostname")) { - LOG.info("Removing property hive_hostname from " + HIVE_ENV_CONFIG); - removeConfigurationPropertiesFromCluster(cluster, HIVE_ENV_CONFIG, Collections.singleton("hive_hostname")); - } - } - } - } - - protected void addSliderClientConfig() throws AmbariException { - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - Clusters clusters = ambariManagementController.getClusters(); - ConfigHelper configHelper = ambariManagementController.getConfigHelper(); - Map clusterMap = getCheckedClusterMap(clusters); - - for (final Cluster cluster : clusterMap.values()) { - Set installedServices = cluster.getServices().keySet(); - if (installedServices.contains(SLIDER_SERVICE_NAME)) { - Config sliderClientConfig = cluster.getDesiredConfigByType(SLIDER_CLIENT_CONFIG); - if (sliderClientConfig == null) { - configHelper.createConfigType(cluster, cluster.getDesiredStackVersion(), - ambariManagementController, SLIDER_CLIENT_CONFIG, new HashMap(), - AUTHENTICATED_USER_NAME, ""); - } - } - } - } - - protected void updateAlerts() { - // map of alert_name -> property_name -> visibility_value - final Map hdfsVisibilityMap = new HashMap(){{ - put("mergeHaMetrics", "HIDDEN"); - put("appId", "HIDDEN"); - put("metricName", "HIDDEN"); - }}; - final Map defaultKeytabVisibilityMap = new HashMap(){{ - put("default.smoke.principal", "HIDDEN"); - put("default.smoke.keytab", "HIDDEN"); - }}; - - final Map percentParameterMap = new HashMap(){{ - put("units", "%"); - put("type", "PERCENT"); - }}; - - Map> visibilityMap = new HashMap>(){{ - put("hive_webhcat_server_status", new HashMap(){{ - put("default.smoke.user", "HIDDEN"); - }}); - put("hive_metastore_process", defaultKeytabVisibilityMap); - put("hive_server_process", defaultKeytabVisibilityMap); - put("zookeeper_server_process", new HashMap(){{ - put("socket.command", "HIDDEN"); - put("socket.command.response", "HIDDEN"); - }}); - }}; - - Map> reportingPercentMap = new HashMap>(){{ - put("hawq_segment_process_percent", percentParameterMap); - put("mapreduce_history_server_cpu", percentParameterMap); - put("yarn_nodemanager_webui_percent", percentParameterMap); - put("yarn_resourcemanager_cpu", percentParameterMap); - put("datanode_process_percent", percentParameterMap); - put("datanode_storage_percent", percentParameterMap); - put("journalnode_process_percent", percentParameterMap); - put("namenode_cpu", percentParameterMap); - put("namenode_hdfs_capacity_utilization", percentParameterMap); - put("datanode_storage", percentParameterMap); - put("datanode_heap_usage", percentParameterMap); - put("storm_supervisor_process_percent", percentParameterMap); - put("hbase_regionserver_process_percent", percentParameterMap); - put("hbase_master_cpu", percentParameterMap); - put("zookeeper_server_process_percent", percentParameterMap); - put("metrics_monitor_process_percent", percentParameterMap); - put("ams_metrics_collector_hbase_master_cpu", percentParameterMap); - }}; - - Map> reportingMultiplierMap = new HashMap>(){{ - put("hawq_segment_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("yarn_nodemanager_webui_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("datanode_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("datanode_storage_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("journalnode_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("storm_supervisor_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("hbase_regionserver_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("zookeeper_server_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - put("metrics_monitor_process_percent", new HashMap() {{ - put("warning", 100); - put("critical", 100); - }}); - }}; - - Map> scriptAlertMultiplierMap = new HashMap>(){{ - put("ambari_agent_disk_usage", new HashMap() {{ - put("percent.used.space.warning.threshold", 100); - put("percent.free.space.critical.threshold", 100); - }}); - put("namenode_last_checkpoint", new HashMap() {{ - put("checkpoint.time.warning.threshold", 100); - put("checkpoint.time.critical.threshold", 100); - }}); - }}; - - String newNameservicePropertyValue = "{{hdfs-site/dfs.internal.nameservices}}"; - final Set alertNamesForNameserviceUpdate = new HashSet() {{ - add("namenode_webui"); - add("namenode_hdfs_blocks_health"); - add("namenode_hdfs_pending_deletion_blocks"); - add("namenode_rpc_latency"); - add("namenode_directory_status"); - add("datanode_health_summary"); - add("namenode_cpu"); - add("namenode_hdfs_capacity_utilization"); - }}; - - // list of alerts that need to get property updates - Set alertNamesForPropertyUpdates = new HashSet() {{ - add("hawq_segment_process_percent"); - add("mapreduce_history_server_cpu"); - add("yarn_nodemanager_webui_percent"); - add("yarn_resourcemanager_cpu"); - add("datanode_process_percent"); - add("datanode_storage_percent"); - add("journalnode_process_percent"); - add("namenode_cpu"); - add("namenode_hdfs_capacity_utilization"); - add("datanode_storage"); - add("datanode_heap_usage"); - add("storm_supervisor_process_percent"); - add("hbase_regionserver_process_percent"); - add("hbase_master_cpu"); - add("zookeeper_server_process_percent"); - add("metrics_monitor_process_percent"); - add("ams_metrics_collector_hbase_master_cpu"); - add("ambari_agent_disk_usage"); - add("namenode_last_checkpoint"); - addAll(alertNamesForNameserviceUpdate); - }}; - - // list of alerts to be removed - Set alertForRemoval = new HashSet() {{ - add("storm_rest_api"); - add("mapreduce_history_server_process"); - }}; - - LOG.info("Updating alert definitions."); - AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); - AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class); - Clusters clusters = ambariManagementController.getClusters(); - - Map clusterMap = getCheckedClusterMap(clusters); - for (final Cluster cluster : clusterMap.values()) { - long clusterID = cluster.getClusterId(); - - // here goes alerts that need get new properties - final AlertDefinitionEntity namenodeLastCheckpointAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "namenode_last_checkpoint"); - final AlertDefinitionEntity namenodeHAHealthAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "namenode_ha_health"); - final AlertDefinitionEntity nodemanagerHealthAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "yarn_nodemanager_health"); - final AlertDefinitionEntity nodemanagerHealthSummaryAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "nodemanager_health_summary"); - final AlertDefinitionEntity hiveMetastoreProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "hive_metastore_process"); - final AlertDefinitionEntity hiveServerProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "hive_server_process"); - final AlertDefinitionEntity hiveWebhcatServerStatusAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "hive_webhcat_server_status"); - final AlertDefinitionEntity flumeAgentStatusAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "flume_agent_status"); - final AlertDefinitionEntity zookeeperServerProcessAlertDefinitionEntity = alertDefinitionDAO.findByName( - clusterID, "zookeeper_server_process"); - - Map> alertDefinitionParams = new HashMap<>(); - checkedPutToMap(alertDefinitionParams, namenodeLastCheckpointAlertDefinitionEntity, - Lists.newArrayList("connection.timeout", "checkpoint.time.warning.threshold", - "checkpoint.time.critical.threshold", "checkpoint.txns.multiplier.warning.threshold", - "checkpoint.txns.multiplier.critical.threshold")); - checkedPutToMap(alertDefinitionParams, namenodeHAHealthAlertDefinitionEntity, - Lists.newArrayList("connection.timeout")); - checkedPutToMap(alertDefinitionParams, nodemanagerHealthAlertDefinitionEntity, - Lists.newArrayList("connection.timeout")); - checkedPutToMap(alertDefinitionParams, nodemanagerHealthSummaryAlertDefinitionEntity, - Lists.newArrayList("connection.timeout")); - checkedPutToMap(alertDefinitionParams, hiveMetastoreProcessAlertDefinitionEntity, - Lists.newArrayList("default.smoke.user", "default.smoke.principal", "default.smoke.keytab")); - checkedPutToMap(alertDefinitionParams, hiveServerProcessAlertDefinitionEntity, - Lists.newArrayList("default.smoke.user", "default.smoke.principal", "default.smoke.keytab")); - checkedPutToMap(alertDefinitionParams, hiveWebhcatServerStatusAlertDefinitionEntity, - Lists.newArrayList("default.smoke.user", "connection.timeout")); - checkedPutToMap(alertDefinitionParams, flumeAgentStatusAlertDefinitionEntity, - Lists.newArrayList("run.directory")); - checkedPutToMap(alertDefinitionParams, zookeeperServerProcessAlertDefinitionEntity, - Lists.newArrayList("socket.command", "socket.command.response")); - - - Map definitionsForPropertyUpdates = new HashMap<>(); - - // adding new properties - for (Map.Entry> entry : alertDefinitionParams.entrySet()){ - AlertDefinitionEntity alertDefinition = entry.getKey(); - String source = alertDefinition.getSource(); - alertDefinition.setSource(addParam(source, entry.getValue())); - definitionsForPropertyUpdates.put(alertDefinition.getDefinitionId(), alertDefinition); - } - - // here goes alerts that need update for existing properties - for (String name : alertNamesForPropertyUpdates) { - AlertDefinitionEntity alertDefinition = alertDefinitionDAO.findByName(clusterID, name); - if (alertDefinition != null && !definitionsForPropertyUpdates.containsKey(alertDefinition.getDefinitionId())) { - definitionsForPropertyUpdates.put(alertDefinition.getDefinitionId(), alertDefinition); - } - } - - // updating old and new properties, best way to use map like visibilityMap. - for (AlertDefinitionEntity alertDefinition : definitionsForPropertyUpdates.values()) { - // here goes property updates - if (visibilityMap.containsKey(alertDefinition.getDefinitionName())) { - for (Map.Entry entry : visibilityMap.get(alertDefinition.getDefinitionName()).entrySet()){ - String paramName = entry.getKey(); - String visibilityValue = entry.getValue(); - String source = alertDefinition.getSource(); - alertDefinition.setSource(addParamOption(source, paramName, "visibility", visibilityValue)); - } - } - // update percent script alerts param values from 0.x to 0.x * 100 values - if (scriptAlertMultiplierMap.containsKey(alertDefinition.getDefinitionName())) { - for (Map.Entry entry : scriptAlertMultiplierMap.get(alertDefinition.getDefinitionName()).entrySet()){ - String paramName = entry.getKey(); - Integer multiplier = entry.getValue(); - String source = alertDefinition.getSource(); - Float oldValue = getParamFloatValue(source, paramName); - if (oldValue == null) { - alertDefinition.setSource(addParam(source, Arrays.asList(paramName))); - } else { - Integer newValue = Math.round(oldValue * multiplier); - alertDefinition.setSource(setParamIntegerValue(source, paramName, newValue)); - } - } - } - - // update reporting alerts(aggregate and metrics) values from 0.x to 0.x * 100 values - if (reportingMultiplierMap.containsKey(alertDefinition.getDefinitionName())) { - for (Map.Entry entry : reportingMultiplierMap.get(alertDefinition.getDefinitionName()).entrySet()){ - String reportingName = entry.getKey(); - Integer multiplier = entry.getValue(); - String source = alertDefinition.getSource(); - Float oldValue = getReportingFloatValue(source, reportingName); - Integer newValue = Math.round(oldValue * multiplier); - alertDefinition.setSource(setReportingIntegerValue(source, reportingName, newValue)); - } - } - - if (reportingPercentMap.containsKey(alertDefinition.getDefinitionName())) { - for (Map.Entry entry : reportingPercentMap.get(alertDefinition.getDefinitionName()).entrySet()){ - String paramName = entry.getKey(); - String paramValue = entry.getValue(); - String source = alertDefinition.getSource(); - alertDefinition.setSource(addReportingOption(source, paramName, paramValue)); - } - } - - if (alertNamesForNameserviceUpdate.contains(alertDefinition.getDefinitionName())) { - String source = alertDefinition.getSource(); - alertDefinition.setSource(setNameservice(source, newNameservicePropertyValue)); - } - // regeneration of hash and writing modified alerts to database, must go after all modifications finished - alertDefinition.setHash(UUID.randomUUID().toString()); - alertDefinitionDAO.merge(alertDefinition); - } - //update Atlas alert - final AlertDefinitionEntity atlasMetadataServerWebUI = alertDefinitionDAO.findByName( - clusterID, "metadata_server_webui"); - if (atlasMetadataServerWebUI != null) { - String source = atlasMetadataServerWebUI.getSource(); - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - - JsonObject uriJson = sourceJson.get("uri").getAsJsonObject(); - uriJson.remove("kerberos_keytab"); - uriJson.remove("kerberos_principal"); - uriJson.addProperty("kerberos_keytab", "{{cluster-env/smokeuser_keytab}}"); - uriJson.addProperty("kerberos_principal", "{{cluster-env/smokeuser_principal_name}}"); - - atlasMetadataServerWebUI.setSource(sourceJson.toString()); - - atlasMetadataServerWebUI.setHash(UUID.randomUUID().toString()); - alertDefinitionDAO.merge(atlasMetadataServerWebUI); - } - - for (String alertName: alertForRemoval) { - AlertDefinitionEntity alertDefinition = alertDefinitionDAO.findByName(clusterID, alertName); - if (alertDefinition != null) { - LOG.info("Removing alert : " + alertName); - alertDefinitionDAO.remove(alertDefinition); - } - } - } - } - - protected String setNameservice(String source, String paramValue) { - final String nameservicePropertyName = "nameservice"; - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - JsonObject highAvailability = sourceJson.getAsJsonObject("uri").getAsJsonObject("high_availability"); - if (highAvailability.has(nameservicePropertyName)) { - highAvailability.addProperty(nameservicePropertyName, paramValue); - } - return sourceJson.toString(); - } - - /* - * Simple put method with check for key is not null - * */ - private void checkedPutToMap(Map> alertDefinitionParams, AlertDefinitionEntity alertDefinitionEntity, - List params) { - if (alertDefinitionEntity != null) { - alertDefinitionParams.put(alertDefinitionEntity, params); - } - } - - /** - * Add option to script parameter. - * @param source json string of script source - * @param paramName parameter name - * @param optionName option name - * @param optionValue option value - * @return modified source - */ - protected String addParamOption(String source, String paramName, String optionName, String optionValue){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - JsonArray parametersJson = sourceJson.getAsJsonArray("parameters"); - if(parametersJson != null && !parametersJson.isJsonNull()) { - for(JsonElement param : parametersJson) { - if(param.isJsonObject()) { - JsonObject paramObject = param.getAsJsonObject(); - if(paramObject.has("name") && paramObject.get("name").getAsString().equals(paramName)){ - paramObject.add(optionName, new JsonPrimitive(optionValue)); - } - } - } - } - return sourceJson.toString(); - } - - /** - * Returns param value as float. - * @param source source of script alert - * @param paramName param name - * @return param value as float - */ - protected Float getParamFloatValue(String source, String paramName){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - JsonArray parametersJson = sourceJson.getAsJsonArray("parameters"); - if(parametersJson != null && !parametersJson.isJsonNull()) { - for(JsonElement param : parametersJson) { - if(param.isJsonObject()) { - JsonObject paramObject = param.getAsJsonObject(); - if(paramObject.has("name") && paramObject.get("name").getAsString().equals(paramName)){ - if(paramObject.has("value")) { - return paramObject.get("value").getAsFloat(); - } - } - } - } - } - return null; - } - - /** - * Set integer param value. - * @param source source of script alert - * @param paramName param name - * @param value new param value - * @return modified source - */ - protected String setParamIntegerValue(String source, String paramName, Integer value){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - JsonArray parametersJson = sourceJson.getAsJsonArray("parameters"); - if(parametersJson != null && !parametersJson.isJsonNull()) { - for(JsonElement param : parametersJson) { - if(param.isJsonObject()) { - JsonObject paramObject = param.getAsJsonObject(); - if(paramObject.has("name") && paramObject.get("name").getAsString().equals(paramName)){ - paramObject.add("value", new JsonPrimitive(value)); - } - } - } - } - return sourceJson.toString(); - } - - /** - * Returns reporting value as float. - * @param source source of aggregate or metric alert - * @param reportingName reporting name, must be "warning" or "critical" - * @return reporting value as float - */ - protected Float getReportingFloatValue(String source, String reportingName){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - return sourceJson.getAsJsonObject("reporting").getAsJsonObject(reportingName).get("value").getAsFloat(); - } - - /** - * Set integer value of reporting. - * @param source source of aggregate or metric alert - * @param reportingName reporting name, must be "warning" or "critical" - * @param value new value - * @return modified source - */ - protected String setReportingIntegerValue(String source, String reportingName, Integer value){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - sourceJson.getAsJsonObject("reporting").getAsJsonObject(reportingName).add("value", new JsonPrimitive(value)); - return sourceJson.toString(); - } - - /** - * Add option to reporting - * @param source source of aggregate or metric alert - * @param optionName option name - * @param value option value - * @return modified source - */ - protected String addReportingOption(String source, String optionName, String value){ - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - sourceJson.getAsJsonObject("reporting").add(optionName, new JsonPrimitive(value)); - return sourceJson.toString(); - } - - protected String addParam(String source, List params) { - JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject(); - JsonArray parametersJson = sourceJson.getAsJsonArray("parameters"); - - boolean parameterExists = parametersJson != null && !parametersJson.isJsonNull(); - - if (parameterExists) { - Iterator jsonElementIterator = parametersJson.iterator(); - while (jsonElementIterator.hasNext()) { - JsonElement element = jsonElementIterator.next(); - JsonElement name = element.getAsJsonObject().get("name"); - if (name != null && !name.isJsonNull() && params.contains(name.getAsString())) { - params.remove(name.getAsString()); - } - } - if (params.size() == 0) { - return sourceJson.toString(); - } - } - - List paramsToAdd = new ArrayList<>(); - - if (params.contains("connection.timeout")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("connection.timeout")); - param.add("display_name", new JsonPrimitive("Connection Timeout")); - param.add("value", new JsonPrimitive(5.0)); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The maximum time before this alert is considered to be CRITICAL")); - param.add("units", new JsonPrimitive("seconds")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - - paramsToAdd.add(param); - - } - if (params.contains("checkpoint.time.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("checkpoint.time.warning.threshold")); - param.add("display_name", new JsonPrimitive("Checkpoint Warning")); - param.add("value", new JsonPrimitive(2.0)); - param.add("type", new JsonPrimitive("PERCENT")); - param.add("description", new JsonPrimitive("The percentage of the last checkpoint time greater than the interval in order to trigger a warning alert.")); - param.add("units", new JsonPrimitive("%")); - param.add("threshold", new JsonPrimitive("WARNING")); - - paramsToAdd.add(param); - - } - if (params.contains("checkpoint.time.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("checkpoint.time.critical.threshold")); - param.add("display_name", new JsonPrimitive("Checkpoint Critical")); - param.add("value", new JsonPrimitive(4.0)); - param.add("type", new JsonPrimitive("PERCENT")); - param.add("description", new JsonPrimitive("The percentage of the last checkpoint time greater than the interval in order to trigger a critical alert.")); - param.add("units", new JsonPrimitive("%")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - - paramsToAdd.add(param); - - } - if (params.contains("checkpoint.txns.multiplier.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("checkpoint.txns.multiplier.warning.threshold")); - param.add("display_name", new JsonPrimitive("Uncommitted transactions Warning")); - param.add("value", new JsonPrimitive(2.0)); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The multiplier to use against dfs.namenode.checkpoint.period compared to the difference between last transaction id and most recent transaction id beyond which to trigger a warning alert.")); - param.add("threshold", new JsonPrimitive("WARNING")); - - paramsToAdd.add(param); - } - if (params.contains("checkpoint.txns.multiplier.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("checkpoint.txns.multiplier.critical.threshold")); - param.add("display_name", new JsonPrimitive("Uncommitted transactions Critical")); - param.add("value", new JsonPrimitive(4.0)); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The multiplier to use against dfs.namenode.checkpoint.period compared to the difference between last transaction id and most recent transaction id beyond which to trigger a critical alert.")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - - paramsToAdd.add(param); - } - if (params.contains("default.smoke.user")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("default.smoke.user")); - param.add("display_name", new JsonPrimitive("Default Smoke User")); - param.add("value", new JsonPrimitive("ambari-qa")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("The user that will run the Hive commands if not specified in cluster-env/smokeuser")); - - paramsToAdd.add(param); - - } - if (params.contains("default.smoke.principal")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("default.smoke.principal")); - param.add("display_name", new JsonPrimitive("Default Smoke Principal")); - param.add("value", new JsonPrimitive("ambari-qa@EXAMPLE.COM")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("The principal to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_principal_name")); - - paramsToAdd.add(param); - - } - if (params.contains("default.smoke.keytab")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("default.smoke.keytab")); - param.add("display_name", new JsonPrimitive("Default Smoke Keytab")); - param.add("value", new JsonPrimitive("/etc/security/keytabs/smokeuser.headless.keytab")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("The keytab to use when retrieving the kerberos ticket if not specified in cluster-env/smokeuser_keytab")); - - paramsToAdd.add(param); - - } - if (params.contains("run.directory")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("run.directory")); - param.add("display_name", new JsonPrimitive("Run Directory")); - param.add("value", new JsonPrimitive("/var/run/flume")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("The directory where flume agent processes will place their PID files.")); - - paramsToAdd.add(param); - - } - if (params.contains("minimum.free.space")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("minimum.free.space")); - param.add("display_name", new JsonPrimitive("Minimum Free Space")); - param.add("value", new JsonPrimitive("5000000000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The overall amount of free disk space left before an alert is triggered.")); - param.add("units", new JsonPrimitive("bytes")); - param.add("threshold", new JsonPrimitive("WARNING")); - paramsToAdd.add(param); - - } - if (params.contains("percent.used.space.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("percent.used.space.warning.threshold")); - param.add("display_name", new JsonPrimitive("Warning")); - param.add("value", new JsonPrimitive("50")); - param.add("type", new JsonPrimitive("PERCENT")); - param.add("description", new JsonPrimitive("The percent of disk space consumed before a warning is triggered.")); - param.add("units", new JsonPrimitive("%")); - param.add("threshold", new JsonPrimitive("WARNING")); - paramsToAdd.add(param); - - } - if (params.contains("percent.free.space.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("percent.free.space.critical.threshold")); - param.add("display_name", new JsonPrimitive("Critical")); - param.add("value", new JsonPrimitive("80")); - param.add("type", new JsonPrimitive("PERCENT")); - param.add("description", new JsonPrimitive("The percent of disk space consumed before a critical alert is triggered.")); - param.add("units", new JsonPrimitive("%")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - paramsToAdd.add(param); - - } - if (params.contains("request.by.status.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("request.by.status.warning.threshold")); - param.add("display_name", new JsonPrimitive("Warning Request Time")); - param.add("value", new JsonPrimitive("3000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to find requests in progress before a warning alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("WARNING")); - paramsToAdd.add(param); - - } - if (params.contains("request.by.status.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("request.by.status.critical.threshold")); - param.add("display_name", new JsonPrimitive("Critical Request Time")); - param.add("value", new JsonPrimitive("5000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to find requests in progress before a critical alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - paramsToAdd.add(param); - - } - if (params.contains("task.status.aggregation.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("task.status.aggregation.warning.threshold")); - param.add("display_name", new JsonPrimitive("Warning Process Time")); - param.add("value", new JsonPrimitive("3000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to calculate a request's status from its tasks before a warning alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("WARNING")); - paramsToAdd.add(param); - - } - if (params.contains("task.status.aggregation.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("task.status.aggregation.critical.threshold")); - param.add("display_name", new JsonPrimitive("Critical Process Time")); - param.add("value", new JsonPrimitive("5000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to calculate a request's status from its tasks before a critical alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - paramsToAdd.add(param); - - } - if (params.contains("rest.api.cluster.warning.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("rest.api.cluster.warning.threshold")); - param.add("display_name", new JsonPrimitive("Warning Response Time")); - param.add("value", new JsonPrimitive("5000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to get a cluster via the REST API before a warning alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("WARNING")); - paramsToAdd.add(param); - - } - if (params.contains("rest.api.cluster.critical.threshold")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("rest.api.cluster.critical.threshold")); - param.add("display_name", new JsonPrimitive("Critical Response Time")); - param.add("value", new JsonPrimitive("7000")); - param.add("type", new JsonPrimitive("NUMERIC")); - param.add("description", new JsonPrimitive("The time to get a cluster via the REST API before a critical alert is triggered.")); - param.add("units", new JsonPrimitive("ms")); - param.add("threshold", new JsonPrimitive("CRITICAL")); - paramsToAdd.add(param); - - } - if (params.contains("socket.command")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("socket.command")); - param.add("display_name", new JsonPrimitive("Socket Command")); - param.add("value", new JsonPrimitive("ruok")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("A socket command which queries ZooKeeper to respond with its state. The expected response is imok.")); - paramsToAdd.add(param); - - } - if (params.contains("socket.command.response")) { - JsonObject param = new JsonObject(); - param.add("name", new JsonPrimitive("socket.command.response")); - param.add("display_name", new JsonPrimitive("Expected Response")); - param.add("value", new JsonPrimitive("imok")); - param.add("type", new JsonPrimitive("STRING")); - param.add("description", new JsonPrimitive("The expected response to the socket command.")); - paramsToAdd.add(param); - - } - - - - if (!parameterExists) { - parametersJson = new JsonArray(); - for (JsonObject param : paramsToAdd) { - parametersJson.add(param); - } - sourceJson.add("parameters", parametersJson); - } else { - for (JsonObject param : paramsToAdd) { - parametersJson.add(param); - } - sourceJson.remove("parameters"); - sourceJson.add("parameters", parametersJson); - } - - return sourceJson.toString(); - } - - protected void updateAdminPermissionTable() throws SQLException { - // Add the sort_order column to the adminpermission table - dbAccessor.addColumn(ADMIN_PERMISSION_TABLE, - new DBColumnInfo(SORT_ORDER_COL, Short.class, null, 1, false)); - - // Add the principal_id column to the adminpermission table - // Note: This is set to nullable here, but will be altered once the column has been set - // properly during the DML update phase. - dbAccessor.addColumn(ADMIN_PERMISSION_TABLE, - new DBColumnInfo(PRINCIPAL_ID_COL, Long.class, null, null, true)); - } - - protected void updateTopologyRequestTable() throws SQLException { - // Add the sort_order column to the adminpermission table - dbAccessor.addColumn(TOPOLOGY_REQUEST_TABLE, - new DBColumnInfo(PROVISION_ACTION_COL, String.class, 255, null, true)); - } - - /** - * Updates the {@value #ALERT_DEFINITION_TABLE} in the following ways: - *
    - *
  • Craetes the {@value #HELP_URL_COLUMN} column
    • - *
  • Craetes the {@value #REPEAT_TOLERANCE_COLUMN} column
    • - *
  • Craetes the {@value #REPEAT_TOLERANCE_ENABLED_COLUMN} column
    • - *
- * - * @throws SQLException - */ - protected void updateAlertDefinitionTable() throws SQLException { - dbAccessor.addColumn(ALERT_DEFINITION_TABLE, - new DBColumnInfo(HELP_URL_COLUMN, String.class, 512, null, true)); - - dbAccessor.addColumn(ALERT_DEFINITION_TABLE, - new DBColumnInfo(REPEAT_TOLERANCE_COLUMN, Integer.class, null, 1, false)); - - dbAccessor.addColumn(ALERT_DEFINITION_TABLE, - new DBColumnInfo(REPEAT_TOLERANCE_ENABLED_COLUMN, Short.class, null, 0, false)); - } - - /** - * Updates the {@value #ALERT_CURRENT_TABLE} in the following ways: - *
    - *
  • Creates the {@value #ALERT_CURRENT_OCCURRENCES_COLUMN} column
    • - *
  • Creates the {@value #ALERT_CURRENT_FIRMNESS_COLUMN} column
    • - *
- * - * @throws SQLException - */ - protected void updateAlertCurrentTable() throws SQLException { - dbAccessor.addColumn(ALERT_CURRENT_TABLE, - new DBColumnInfo(ALERT_CURRENT_OCCURRENCES_COLUMN, Long.class, null, 1, false)); - - dbAccessor.addColumn(ALERT_CURRENT_TABLE, new DBColumnInfo(ALERT_CURRENT_FIRMNESS_COLUMN, - String.class, 255, AlertFirmness.HARD.name(), false)); - } - - /** - * Updates the {@value #ALERT_TARGET_TABLE} in the following ways: - *
    - *
  • Creates the {@value #ALERT_TARGET_ENABLED_COLUMN} column
    • - *
- * - * @throws SQLException - */ - protected void updateAlertTargetTable() throws SQLException { - dbAccessor.addColumn(ALERT_TARGET_TABLE, - new DBColumnInfo(ALERT_TARGET_ENABLED_COLUMN, Short.class, null, 1, false)); - } - - protected void setRoleSortOrder() throws SQLException { - String updateStatement = "UPDATE " + ADMIN_PERMISSION_TABLE + " SET " + SORT_ORDER_COL + "=%d WHERE " + PERMISSION_ID_COL + "='%s'"; - - LOG.info("Setting permission labels"); - dbAccessor.executeUpdate(String.format(updateStatement, - 1, PermissionEntity.AMBARI_ADMINISTRATOR_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 2, PermissionEntity.CLUSTER_ADMINISTRATOR_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 3, PermissionEntity.CLUSTER_OPERATOR_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 4, PermissionEntity.SERVICE_ADMINISTRATOR_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 5, PermissionEntity.SERVICE_OPERATOR_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 6, PermissionEntity.CLUSTER_USER_PERMISSION_NAME)); - dbAccessor.executeUpdate(String.format(updateStatement, - 7, PermissionEntity.VIEW_USER_PERMISSION_NAME)); - } - - /** - * Create and update records to create the role-based principals. - *

- * This includes creating the new "ROLE" principal type, a principal for each role, and finally - * updating the princial_id column for the role. - */ - void createRolePrincipals() throws SQLException { - // Create Role Principal Type - PrincipalTypeEntity rolePrincipalType = new PrincipalTypeEntity(); - rolePrincipalType.setName("ROLE"); - - // creates the new record and returns an entity with the id set. - rolePrincipalType = principalTypeDAO.merge(rolePrincipalType); - - // Get the roles (adminpermissions) and create a principal for each.... set the role's principal_id - // value as we go... - List roleEntities = permissionDAO.findAll(); - - for (PermissionEntity roleEntity : roleEntities) { - PrincipalEntity principalEntity = new PrincipalEntity(); - principalEntity.setPrincipalType(rolePrincipalType); - - roleEntity.setPrincipal(principalDAO.merge(principalEntity)); - - permissionDAO.merge(roleEntity); - } - - // Fix the adminpermission.principal_id column to be non-nullable: - dbAccessor.alterColumn(ADMIN_PERMISSION_TABLE, - new DBColumnInfo(PRINCIPAL_ID_COL, Long.class, null, null, false)); - } - - /** - * Makes the following changes to the {@value #REPO_VERSION_TABLE} table: - *

    - *
  • repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL
    • - *
  • version_url VARCHAR(1024)
    • - *
  • version_xml MEDIUMTEXT
    • - *
  • version_xsd VARCHAR(512)
    • - *
  • parent_id BIGINT
    • - *
- * - * @throws SQLException - */ - private void updateRepoVersionTableDDL() throws SQLException { - DBColumnInfo repoTypeColumn = new DBColumnInfo("repo_type", String.class, 255, RepositoryType.STANDARD.name(), false); - DBColumnInfo versionUrlColumn = new DBColumnInfo("version_url", String.class, 1024, null, true); - DBColumnInfo versionXmlColumn = new DBColumnInfo("version_xml", Clob.class, null, null, true); - DBColumnInfo versionXsdColumn = new DBColumnInfo("version_xsd", String.class, 512, null, true); - DBColumnInfo parentIdColumn = new DBColumnInfo("parent_id", Long.class, null, null, true); - - dbAccessor.addColumn(REPO_VERSION_TABLE, repoTypeColumn); - dbAccessor.addColumn(REPO_VERSION_TABLE, versionUrlColumn); - dbAccessor.addColumn(REPO_VERSION_TABLE, versionXmlColumn); - dbAccessor.addColumn(REPO_VERSION_TABLE, versionXsdColumn); - dbAccessor.addColumn(REPO_VERSION_TABLE, parentIdColumn); - } - - /** - * Makes the following changes to the {@value #SERVICE_COMPONENT_DS_TABLE} table, - * but only if the table doesn't have it's new PK set. - *
    - *
  • id BIGINT NOT NULL
    • - *
  • Drops FKs on {@value #HOST_COMPONENT_DS_TABLE} and {@value #HOST_COMPONENT_STATE_TABLE}
    • - *
  • Populates ID in {@value #SERVICE_COMPONENT_DS_TABLE}
    • - *
  • Creates {@code UNIQUE} constraint on {@value #HOST_COMPONENT_DS_TABLE}
    • - *
  • Adds FKs on {@value #HOST_COMPONENT_DS_TABLE} and {@value #HOST_COMPONENT_STATE_TABLE}
    • - *
  • Adds new sequence value of {@code servicecomponentdesiredstate_id_seq}
    • - *
- * - * @throws SQLException - */ - void updateServiceComponentDesiredStateTableDDL() throws SQLException { - if (dbAccessor.tableHasPrimaryKey(SERVICE_COMPONENT_DS_TABLE, ID)) { - LOG.info("Skipping {} table Primary Key modifications since the new {} column already exists", - SERVICE_COMPONENT_DS_TABLE, ID); - - return; - } - - // drop FKs to SCDS in both HCDS and HCS tables - // These are the expected constraint names - dbAccessor.dropFKConstraint(HOST_COMPONENT_DS_TABLE, "hstcmpnntdesiredstatecmpnntnme"); - dbAccessor.dropFKConstraint(HOST_COMPONENT_STATE_TABLE, "hstcomponentstatecomponentname"); - // These are the old (pre Ambari 1.5) constraint names, however still found on some installations - dbAccessor.dropFKConstraint(HOST_COMPONENT_DS_TABLE, "FK_hostcomponentdesiredstate_component_name"); - dbAccessor.dropFKConstraint(HOST_COMPONENT_STATE_TABLE, "FK_hostcomponentstate_component_name"); - - // remove existing compound PK - dbAccessor.dropPKConstraint(SERVICE_COMPONENT_DS_TABLE, "servicecomponentdesiredstate_pkey"); - - // add new PK column to SCDS, making it nullable for now - DBColumnInfo idColumn = new DBColumnInfo(ID, Long.class, null, null, true); - dbAccessor.addColumn(SERVICE_COMPONENT_DS_TABLE, idColumn); - - // populate SCDS id column - AtomicLong scdsIdCounter = new AtomicLong(1); - Statement statement = null; - ResultSet resultSet = null; - try { - statement = dbAccessor.getConnection().createStatement(); - if (statement != null) { - String selectSQL = String.format("SELECT cluster_id, service_name, component_name FROM %s", - SERVICE_COMPONENT_DS_TABLE); - - resultSet = statement.executeQuery(selectSQL); - while (null != resultSet && resultSet.next()) { - final Long clusterId = resultSet.getLong("cluster_id"); - final String serviceName = resultSet.getString("service_name"); - final String componentName = resultSet.getString("component_name"); - - String updateSQL = String.format( - "UPDATE %s SET %s = %d WHERE cluster_id = %d AND service_name = '%s' AND component_name = '%s'", - SERVICE_COMPONENT_DS_TABLE, ID, scdsIdCounter.getAndIncrement(), clusterId, - serviceName, componentName); - - dbAccessor.executeQuery(updateSQL); - } - } - } finally { - JdbcUtils.closeResultSet(resultSet); - JdbcUtils.closeStatement(statement); - } - - // make the column NON NULL now - dbAccessor.alterColumn(SERVICE_COMPONENT_DS_TABLE, - new DBColumnInfo(ID, Long.class, null, null, false)); - - // create a new PK, matching the name of the constraint found in SQL - dbAccessor.addPKConstraint(SERVICE_COMPONENT_DS_TABLE, "pk_sc_desiredstate", ID); - - // create UNIQUE constraint, ensuring column order matches SQL files - String[] uniqueColumns = new String[] { "component_name", "service_name", "cluster_id" }; - dbAccessor.addUniqueConstraint(SERVICE_COMPONENT_DS_TABLE, "UQ_scdesiredstate_name", - uniqueColumns); - - // add FKs back to SCDS in both HCDS and HCS tables - dbAccessor.addFKConstraint(HOST_COMPONENT_DS_TABLE, "hstcmpnntdesiredstatecmpnntnme", - uniqueColumns, SERVICE_COMPONENT_DS_TABLE, uniqueColumns, false); - - dbAccessor.addFKConstraint(HOST_COMPONENT_STATE_TABLE, "hstcomponentstatecomponentname", - uniqueColumns, SERVICE_COMPONENT_DS_TABLE, uniqueColumns, false); - - // Add sequence for SCDS id - addSequence("servicecomponentdesiredstate_id_seq", scdsIdCounter.get(), false); - } - - /** - * Makes the following changes to the {@value #SERVICE_COMPONENT_HISTORY_TABLE} table: - *
    - *
  • id BIGINT NOT NULL
    • - *
  • component_id BIGINT NOT NULL
    • - *
  • upgrade_id BIGINT NOT NULL
    • - *
  • from_stack_id BIGINT NOT NULL
    • - *
  • to_stack_id BIGINT NOT NULL
    • - *
  • CONSTRAINT PK_sc_history PRIMARY KEY (id)
    • - *
  • CONSTRAINT FK_sc_history_component_id FOREIGN KEY (component_id) REFERENCES servicecomponentdesiredstate (id)
    • - *
  • CONSTRAINT FK_sc_history_upgrade_id FOREIGN KEY (upgrade_id) REFERENCES upgrade (upgrade_id)
    • - *
  • CONSTRAINT FK_sc_history_from_stack_id FOREIGN KEY (from_stack_id) REFERENCES stack (stack_id)
    • - *
  • CONSTRAINT FK_sc_history_to_stack_id FOREIGN KEY (to_stack_id) REFERENCES stack (stack_id)
    • - *
  • Creates the {@code servicecomponent_history_id_seq}
    • - *
- * - * @throws SQLException - */ - private void createServiceComponentHistoryTable() throws SQLException { - List columns = new ArrayList<>(); - columns.add(new DBColumnInfo(ID, Long.class, null, null, false)); - columns.add(new DBColumnInfo("component_id", Long.class, null, null, false)); - columns.add(new DBColumnInfo("upgrade_id", Long.class, null, null, false)); - columns.add(new DBColumnInfo("from_stack_id", Long.class, null, null, false)); - columns.add(new DBColumnInfo("to_stack_id", Long.class, null, null, false)); - dbAccessor.createTable(SERVICE_COMPONENT_HISTORY_TABLE, columns, (String[]) null); - - dbAccessor.addPKConstraint(SERVICE_COMPONENT_HISTORY_TABLE, "PK_sc_history", ID); - - dbAccessor.addFKConstraint(SERVICE_COMPONENT_HISTORY_TABLE, "FK_sc_history_component_id", - "component_id", SERVICE_COMPONENT_DS_TABLE, "id", false); - - dbAccessor.addFKConstraint(SERVICE_COMPONENT_HISTORY_TABLE, "FK_sc_history_upgrade_id", - "upgrade_id", UPGRADE_TABLE, "upgrade_id", false); - - dbAccessor.addFKConstraint(SERVICE_COMPONENT_HISTORY_TABLE, "FK_sc_history_from_stack_id", - "from_stack_id", STACK_TABLE, "stack_id", false); - - dbAccessor.addFKConstraint(SERVICE_COMPONENT_HISTORY_TABLE, "FK_sc_history_to_stack_id", - "to_stack_id", STACK_TABLE, "stack_id", false); - - addSequence("servicecomponent_history_id_seq", 0L, false); - } - - /** - * Alter servicecomponentdesiredstate table to add recovery_enabled column. - * @throws SQLException - */ - private void updateServiceComponentDesiredStateTable() throws SQLException { - // ALTER TABLE servicecomponentdesiredstate ADD COLUMN - // recovery_enabled SMALLINT DEFAULT 0 NOT NULL - dbAccessor.addColumn(SERVICE_COMPONENT_DESIRED_STATE_TABLE, - new DBColumnInfo(RECOVERY_ENABLED_COL, Short.class, null, 0, false)); - - dbAccessor.addColumn(SERVICE_COMPONENT_DESIRED_STATE_TABLE, - new DBColumnInfo(DESIRED_VERSION_COLUMN_NAME, String.class, 255, State.UNKNOWN.toString(), false)); - } - - /** - * Alter host_role_command table to add original_start_time, which is needed because the start_time column now - * allows overriding the value in ActionScheduler.java - * @throws SQLException - */ - private void updateHostRoleCommandTableDDL() throws SQLException { - final String columnName = "original_start_time"; - DBColumnInfo originalStartTimeColumn = new DBColumnInfo(columnName, Long.class, null, -1L, true); - dbAccessor.addColumn(HOST_ROLE_COMMAND_TABLE, originalStartTimeColumn); - } - - /** - * Alter host_role_command table to update original_start_time with values and make it non-nullable - * @throws SQLException - */ - protected void updateHostRoleCommandTableDML() throws SQLException { - final String columnName = "original_start_time"; - dbAccessor.executeQuery("UPDATE " + HOST_ROLE_COMMAND_TABLE + " SET original_start_time = start_time", false); - dbAccessor.executeQuery("UPDATE " + HOST_ROLE_COMMAND_TABLE + " SET original_start_time=-1 WHERE original_start_time IS NULL"); - dbAccessor.setColumnNullable(HOST_ROLE_COMMAND_TABLE, columnName, false); - } - - /** - * Puts each item in the specified list inside single quotes and - * returns a comma separated value for use in a SQL query. - * @param list - * @return - */ - private String sqlStringListFromArrayList(List list) { - List sqlList = new ArrayList<>(list.size()); - - for (String item : list) { - sqlList.add(String.format("'%s'", item.trim())); - } - - return StringUtils.join(sqlList, ','); - } - - /** - * Update clusterconfig table for config type 'cluster-env' with the - * recovery attributes. - * - * @throws AmbariException - */ - private void updateRecoveryClusterEnvConfig() throws AmbariException { - Map propertyMap = new HashMap<>(); - - if (StringUtils.isNotEmpty(config.getNodeRecoveryType())) { - propertyMap.put(RecoveryConfigHelper.RECOVERY_ENABLED_KEY, "true"); - propertyMap.put(RecoveryConfigHelper.RECOVERY_TYPE_KEY, config.getNodeRecoveryType()); - } - else { - propertyMap.put(RecoveryConfigHelper.RECOVERY_ENABLED_KEY, "false"); - } - - if (StringUtils.isNotEmpty(config.getNodeRecoveryLifetimeMaxCount())) { - propertyMap.put(RecoveryConfigHelper.RECOVERY_LIFETIME_MAX_COUNT_KEY, config.getNodeRecoveryLifetimeMaxCount()); - } - - if (StringUtils.isNotEmpty(config.getNodeRecoveryMaxCount())) { - propertyMap.put(RecoveryConfigHelper.RECOVERY_MAX_COUNT_KEY, config.getNodeRecoveryMaxCount()); - } - - if (StringUtils.isNotEmpty(config.getNodeRecoveryRetryGap())) { - propertyMap.put(RecoveryConfigHelper.RECOVERY_RETRY_GAP_KEY, config.getNodeRecoveryRetryGap()); - } - - if (StringUtils.isNotEmpty(config.getNodeRecoveryWindowInMin())) { - propertyMap.put(RecoveryConfigHelper.RECOVERY_WINDOW_IN_MIN_KEY, config.getNodeRecoveryWindowInMin()); - } - - AmbariManagementController ambariManagementController = injector.getInstance( - AmbariManagementController.class); - - Clusters clusters = ambariManagementController.getClusters(); - - // for each cluster, update/create the cluster-env config type in clusterconfig - Map clusterMap = getCheckedClusterMap(clusters); - for (final Cluster cluster : clusterMap.values()) { - updateConfigurationPropertiesForCluster(cluster, ConfigHelper.CLUSTER_ENV, propertyMap, - true /* update if exists */, true /* create new config type */); - } - } - - /** - * Alter servicecomponentdesiredstate table to update recovery_enabled to 1 - * for the components that have been marked for auto start in ambari.properties - * @throws SQLException - */ - private void updateRecoveryComponents() throws SQLException { - - /* - * Whether specific components are enabled/disabled for recovery. Being enabled takes - * precedence over being disabled. When specific components are enabled then only - * those components are enabled. When specific components are disabled then all of - * the other components are enabled. - */ - String enabledComponents = config.getRecoveryEnabledComponents(); - String disabledComponents = config.getRecoveryDisabledComponents(); - String query; - - if (StringUtils.isEmpty(enabledComponents)) { - if (StringUtils.isEmpty(disabledComponents)) { - // disable all components - query = String.format("UPDATE %s SET recovery_enabled = 0", SERVICE_COMPONENT_DESIRED_STATE_TABLE); - } - else { - // enable (1 - disabledComponents) - List disabledComponentsList = Arrays.asList(disabledComponents.split(",")); - String components = sqlStringListFromArrayList(disabledComponentsList); - query = String.format("UPDATE %s SET recovery_enabled = 1 WHERE component_name NOT IN (%s)", - SERVICE_COMPONENT_DESIRED_STATE_TABLE, components); - } - } - else { - // enable the specified components - List enabledComponentsList = Arrays.asList(enabledComponents.split(",")); - String components = sqlStringListFromArrayList(enabledComponentsList); - query = String.format("UPDATE %s SET recovery_enabled = 1 WHERE component_name IN (%s)", - SERVICE_COMPONENT_DESIRED_STATE_TABLE, components); - } - - dbAccessor.executeQuery(query); - } - - - /** - * Update clusterconfig table and servicecomponentdesiredstate table with the - * recovery attributes and componenents to be recovered. - * - * @throws SQLException - */ - @Transactional - protected void updateRecoveryConfigurationDML() throws SQLException, AmbariException { - updateRecoveryClusterEnvConfig(); - updateRecoveryComponents(); - } - - /** - * Update Clusters and Hosts Version State from UPGRADING, UPGRADE_FAILED to INSTALLED - * and UPGRADED to CURRENT if repo_version_id from cluster_version equals repo_version_id of Clusters and Hosts Version State - * - * @throws SQLException - */ - - @Transactional - protected void updateClustersAndHostsVersionStateTableDML() throws SQLException, AmbariException { - - dbAccessor.executeQuery("UPDATE " + HOST_VERSION_TABLE + " SET state = 'INSTALLED' WHERE state IN ('UPGRADING', 'UPGRADE_FAILED', 'UPGRADED')"); - dbAccessor.executeQuery("UPDATE " + CLUSTER_VERSION_TABLE + " SET state = 'INSTALLED' WHERE state IN ('UPGRADING', 'UPGRADE_FAILED', 'UPGRADED')"); - - Statement statement = null; - ResultSet resultSet = null; - try { - statement = dbAccessor.getConnection().createStatement(); - if (statement != null) { - String selectSQL = String.format("SELECT repo_version_id, cluster_id FROM %s WHERE state = 'CURRENT'", - CLUSTER_VERSION_TABLE); - - resultSet = statement.executeQuery(selectSQL); - Set clusterIds = new HashSet<>(); - while (null != resultSet && resultSet.next()) { - Long clusterId = resultSet.getLong("cluster_id"); - if (clusterIds.contains(clusterId)) { - throw new AmbariExcept