Mailing-List: contact dev-help@cloudstack.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@cloudstack.apache.org
From: rhtyd <git@git.apache.org>
To: dev@cloudstack.apache.org
Reply-To: dev@cloudstack.apache.org
References: <git-pr-1502-cloudstack@git.apache.org>
In-Reply-To: <git-pr-1502-cloudstack@git.apache.org>
Subject: [GitHub] cloudstack pull request: CLOUDSTACK-9299: Out-of-band Management f...
Content-Type: text/plain
Message-Id: <20160503074221.5A238DFDCF@git1-us-west.apache.org>
Date: Tue,  3 May 2016 07:42:21 +0000 (UTC)
archived-at: Tue, 03 May 2016 07:42:24 -0000

Github user rhtyd commented on a diff in the pull request:

    https://github.com/apache/cloudstack/pull/1502#discussion_r61847246
  
    --- Diff: server/src/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementServiceImpl.java ---
    @@ -0,0 +1,532 @@
    +// Licensed to the Apache Software Foundation (ASF) under one
    +// or more contributor license agreements.  See the NOTICE file
    +// distributed with this work for additional information
    +// regarding copyright ownership.  The ASF licenses this file
    +// to you under the Apache License, Version 2.0 (the
    +// "License"); you may not use this file except in compliance
    +// with the License.  You may obtain a copy of the License at
    +//
    +//   http://www.apache.org/licenses/LICENSE-2.0
    +//
    +// Unless required by applicable law or agreed to in writing,
    +// software distributed under the License is distributed on an
    +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    +// KIND, either express or implied.  See the License for the
    +// specific language governing permissions and limitations
    +// under the License.
    +package org.apache.cloudstack.outofbandmanagement;
    +
    +import com.cloud.alert.AlertManager;
    +import com.cloud.dc.ClusterDetailsDao;
    +import com.cloud.dc.ClusterDetailsVO;
    +import com.cloud.dc.DataCenter;
    +import com.cloud.dc.DataCenterDetailVO;
    +import com.cloud.dc.dao.DataCenterDetailsDao;
    +import com.cloud.domain.Domain;
    +import com.cloud.event.ActionEvent;
    +import com.cloud.event.ActionEventUtils;
    +import com.cloud.event.EventTypes;
    +import com.cloud.host.Host;
    +import com.cloud.host.dao.HostDao;
    +import com.cloud.org.Cluster;
    +import com.cloud.utils.component.Manager;
    +import com.cloud.utils.component.ManagerBase;
    +import com.cloud.utils.db.GlobalLock;
    +import com.cloud.utils.db.Transaction;
    +import com.cloud.utils.db.TransactionCallback;
    +import com.cloud.utils.db.TransactionStatus;
    +import com.cloud.utils.exception.CloudRuntimeException;
    +import com.cloud.utils.fsm.NoTransitionException;
    +import com.google.common.base.Strings;
    +import com.google.common.cache.Cache;
    +import com.google.common.cache.CacheBuilder;
    +import com.google.common.collect.ImmutableMap;
    +import org.apache.cloudstack.api.response.OutOfBandManagementResponse;
    +import org.apache.cloudstack.context.CallContext;
    +import org.apache.cloudstack.framework.config.ConfigKey;
    +import org.apache.cloudstack.framework.config.Configurable;
    +import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
    +import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverChangePasswordCommand;
    +import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverPowerCommand;
    +import org.apache.cloudstack.outofbandmanagement.driver.OutOfBandManagementDriverResponse;
    +import org.apache.cloudstack.utils.identity.ManagementServerNode;
    +import org.apache.log4j.Logger;
    +import org.springframework.stereotype.Component;
    +
    +import javax.ejb.Local;
    +import javax.inject.Inject;
    +import javax.naming.ConfigurationException;
    +import java.util.ArrayList;
    +import java.util.Collections;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.concurrent.ArrayBlockingQueue;
    +import java.util.concurrent.ExecutorService;
    +import java.util.concurrent.ThreadPoolExecutor;
    +import java.util.concurrent.TimeUnit;
    +
    +@Component
    +@Local(value = {OutOfBandManagementService.class})
    +public class OutOfBandManagementServiceImpl extends ManagerBase implements OutOfBandManagementService, Manager, Configurable {
    +    public static final Logger LOG = Logger.getLogger(OutOfBandManagementServiceImpl.class);
    +
    +    @Inject
    +    private ClusterDetailsDao clusterDetailsDao;
    +    @Inject
    +    private DataCenterDetailsDao dataCenterDetailsDao;
    +    @Inject
    +    private OutOfBandManagementDao outOfBandManagementDao;
    +    @Inject
    +    private HostDao hostDao;
    +    @Inject
    +    private AlertManager alertMgr;
    +
    +    private String name;
    +    private long serviceId;
    +
    +    private List<OutOfBandManagementDriver> outOfBandManagementDrivers = new ArrayList<>();
    +    private Map<String, OutOfBandManagementDriver> outOfBandManagementDriversMap = new HashMap<String, OutOfBandManagementDriver>();
    +
    +    private static final String OOBM_ENABLED_DETAIL = "outOfBandManagementEnabled";
    +    private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST = 120;
    +
    +    private Cache<Long, Long> hostAlertCache;
    +    private static ExecutorService backgroundSyncExecutor;
    +
    +    private String getOutOfBandManagementHostLock(long id) {
    +        return "oobm.host." + id;
    +    }
    +
    +    private void initializeDriversMap() {
    +        if (outOfBandManagementDriversMap.isEmpty() && outOfBandManagementDrivers != null && outOfBandManagementDrivers.size() > 0) {
    +            for (final OutOfBandManagementDriver driver : outOfBandManagementDrivers) {
    +                outOfBandManagementDriversMap.put(driver.getName().toLowerCase(), driver);
    +            }
    +            LOG.debug("Discovered out-of-band management drivers configured in the OutOfBandManagementService");
    +        }
    +    }
    +
    +    private OutOfBandManagementDriver getDriver(final OutOfBandManagement outOfBandManagementConfig) {
    +        if (!Strings.isNullOrEmpty(outOfBandManagementConfig.getDriver())) {
    +            final OutOfBandManagementDriver driver = outOfBandManagementDriversMap.get(outOfBandManagementConfig.getDriver());
    +            if (driver != null) {
    +                return driver;
    +            }
    +        }
    +        throw new CloudRuntimeException("Configured out-of-band management driver is not available. Aborting any out-of-band management action.");
    +    }
    +
    +    protected OutOfBandManagement updateConfig(final OutOfBandManagement outOfBandManagementConfig, final ImmutableMap<OutOfBandManagement.Option, String> options) {
    +        if (outOfBandManagementConfig == null) {
    +            throw new CloudRuntimeException("Out-of-band management is not configured for the host. Aborting.");
    +        }
    +        if (options == null) {
    +            return outOfBandManagementConfig;
    +        }
    +        for (OutOfBandManagement.Option option: options.keySet()) {
    +            final String value = options.get(option);
    +            if (Strings.isNullOrEmpty(value)) {
    +                continue;
    +            }
    +            switch (option) {
    +                case DRIVER:
    +                    outOfBandManagementConfig.setDriver(value);
    +                    break;
    +                case ADDRESS:
    +                    outOfBandManagementConfig.setAddress(value);
    +                    break;
    +                case PORT:
    +                    outOfBandManagementConfig.setPort(Integer.parseInt(value));
    +                    break;
    +                case USERNAME:
    +                    outOfBandManagementConfig.setUsername(value);
    +                    break;
    +                case PASSWORD:
    +                    outOfBandManagementConfig.setPassword(value);
    +                    break;
    +            }
    +        }
    +        return outOfBandManagementConfig;
    +    }
    +
    +    protected ImmutableMap<OutOfBandManagement.Option, String> getOptions(final OutOfBandManagement outOfBandManagementConfig) {
    +        final ImmutableMap.Builder<OutOfBandManagement.Option, String> optionsBuilder = ImmutableMap.builder();
    +        if (outOfBandManagementConfig == null) {
    +            throw new CloudRuntimeException("Out-of-band management is not configured for the host. Aborting.");
    +        }
    +        for (OutOfBandManagement.Option option: OutOfBandManagement.Option.values()) {
    +            String value = null;
    +            switch (option) {
    +                case DRIVER:
    +                    value = outOfBandManagementConfig.getDriver();
    +                    break;
    +                case ADDRESS:
    +                    value = outOfBandManagementConfig.getAddress();
    +                    break;
    +                case PORT:
    +                    if (outOfBandManagementConfig.getPort() != null) {
    +                        value = String.valueOf(outOfBandManagementConfig.getPort());
    +                    }
    +                    break;
    +                case USERNAME:
    +                    value = outOfBandManagementConfig.getUsername();
    +                    break;
    +                case PASSWORD:
    +                    value = outOfBandManagementConfig.getPassword();
    +                    break;
    +            }
    +            if (value != null) {
    +                optionsBuilder.put(option, value);
    +            }
    +        }
    +        return optionsBuilder.build();
    +    }
    +
    +    private void sendAuthError(final Host host, final String message) {
    +        try {
    +            hostAlertCache.asMap().putIfAbsent(host.getId(), 0L);
    +            Long sentCount = hostAlertCache.asMap().get(host.getId());
    +            if (sentCount != null && sentCount <= 0) {
    +                boolean concurrentUpdateResult = hostAlertCache.asMap().replace(host.getId(), sentCount, sentCount+1L);
    +                if (concurrentUpdateResult) {
    +                    final String subject = String.format("Out-of-band management auth-error detected for host:%d in cluster:%d, zone:%d", host.getId(), host.getClusterId(), host.getDataCenterId());
    +                    LOG.error(subject + ": " + message);
    +                    alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_OOBM_AUTH_ERROR, host.getDataCenterId(), host.getPodId(), subject, message);
    +                }
    +            }
    +        } catch (Exception ignored) {
    +        }
    +    }
    +
    +    private boolean transitionPowerState(OutOfBandManagement.PowerState.Event event, OutOfBandManagement outOfBandManagementHost) {
    +        if (outOfBandManagementHost == null) {
    +            return false;
    +        }
    +        OutOfBandManagement.PowerState currentPowerState = outOfBandManagementHost.getPowerState();
    +        try {
    +            OutOfBandManagement.PowerState newPowerState = OutOfBandManagement.PowerState.getStateMachine().getNextState(currentPowerState, event);
    +            boolean result = outOfBandManagementDao.updateState(currentPowerState, event, newPowerState, outOfBandManagementHost, null);
    +            if (result) {
    +                final String message = String.format("Transitioned out-of-band management power state from:%s to:%s due to event:%s for the host id:%d", currentPowerState, newPowerState, event, outOfBandManagementHost.getHostId());
    +                LOG.debug(message);
    +                ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN,
    +                        EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_POWERSTATE_TRANSITION, message);
    +            }
    +            return result;
    +        } catch (NoTransitionException ignored) {
    +            LOG.trace(String.format("Unable to transition out-of-band management power state for host id=%s for the event=%s and current power state=%s", outOfBandManagementHost.getHostId(), event, currentPowerState));
    +        }
    +        return false;
    +    }
    +
    +    private boolean isOutOfBandManagementEnabledForZone(Long zoneId) {
    +        if (zoneId == null) {
    +            return true;
    +        }
    +        final DataCenterDetailVO zoneDetails = dataCenterDetailsDao.findDetail(zoneId, OOBM_ENABLED_DETAIL);
    +        if (zoneDetails != null && !Strings.isNullOrEmpty(zoneDetails.getValue()) && !Boolean.valueOf(zoneDetails.getValue())) {
    +            return false;
    +        }
    +        return true;
    +    }
    +
    +    private boolean isOutOfBandManagementEnabledForCluster(Long clusterId) {
    +        if (clusterId == null) {
    +            return true;
    +        }
    +        final ClusterDetailsVO clusterDetails = clusterDetailsDao.findDetail(clusterId, OOBM_ENABLED_DETAIL);
    +        if (clusterDetails != null && !Strings.isNullOrEmpty(clusterDetails.getValue()) && !Boolean.valueOf(clusterDetails.getValue())) {
    +            return false;
    +        }
    +        return true;
    +    }
    +
    +    private boolean isOutOfBandManagementEnabledForHost(Long hostId) {
    +        if (hostId == null) {
    +            return false;
    +        }
    +        final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(hostId);
    +        if (outOfBandManagementConfig == null || !outOfBandManagementConfig.isEnabled()) {
    +            return false;
    +        }
    +        return true;
    +    }
    +
    +    private void checkOutOfBandManagementEnabledByZoneClusterHost(final Host host) {
    +        if (!isOutOfBandManagementEnabledForZone(host.getDataCenterId())) {
    +            throw new CloudRuntimeException("Out-of-band management is disabled for the host's zone. Aborting Operation.");
    +        }
    +        if (!isOutOfBandManagementEnabledForCluster(host.getClusterId())) {
    +            throw new CloudRuntimeException("Out-of-band management is disabled for the host's cluster. Aborting Operation.");
    +        }
    +        if (!isOutOfBandManagementEnabledForHost(host.getId())) {
    +            throw new CloudRuntimeException("Out-of-band management is disabled or not configured for the host. Aborting Operation.");
    +        }
    +    }
    +
    +    public boolean isOutOfBandManagementEnabled(final Host host) {
    +        return isOutOfBandManagementEnabledForZone(host.getDataCenterId())
    +                && isOutOfBandManagementEnabledForCluster(host.getClusterId())
    +                && isOutOfBandManagementEnabledForHost(host.getId());
    +    }
    +
    +    public boolean transitionPowerStateToDisabled(List<? extends Host> hosts) {
    +        boolean result = true;
    +        for (Host host : hosts) {
    +            result = result && transitionPowerState(OutOfBandManagement.PowerState.Event.Disabled,
    +                    outOfBandManagementDao.findByHost(host.getId()));
    +        }
    +        return result;
    +    }
    +
    +    public void submitBackgroundPowerSyncTask(final Host host) {
    +        if (host != null) {
    +            // Note: This is a blocking queue based executor
    +            backgroundSyncExecutor.submit(new OutOfBandManagementBackgroundTask(this, host, OutOfBandManagement.PowerOperation.STATUS));
    +        }
    +    }
    +
    +    private OutOfBandManagementResponse buildEnableDisableResponse(final boolean enabled) {
    +        final OutOfBandManagementResponse response = new OutOfBandManagementResponse();
    +        response.setEnabled(enabled);
    +        response.setSuccess(true);
    +        return response;
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLEDISABLE, eventDescription = "enabling/disabling out-of-band management on a zone")
    +    public OutOfBandManagementResponse enableDisableOutOfBandManagement(final DataCenter zone, final boolean enabled) {
    +        dataCenterDetailsDao.persist(zone.getId(), OOBM_ENABLED_DETAIL, String.valueOf(enabled));
    +        if (!enabled) {
    +            transitionPowerStateToDisabled(hostDao.findByDataCenterId(zone.getId()));
    +        }
    +        return buildEnableDisableResponse(enabled);
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLEDISABLE, eventDescription = "enabling/disabling out-of-band management on a cluster")
    +    public OutOfBandManagementResponse enableDisableOutOfBandManagement(final Cluster cluster, final boolean enabled) {
    +        clusterDetailsDao.persist(cluster.getId(), OOBM_ENABLED_DETAIL, String.valueOf(enabled));
    +        if (!enabled) {
    +            transitionPowerStateToDisabled(hostDao.findByClusterId(cluster.getId()));
    +        }
    +        return buildEnableDisableResponse(enabled);
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ENABLEDISABLE, eventDescription = "enabling/disabling out-of-band management on a host")
    +    public OutOfBandManagementResponse enableDisableOutOfBandManagement(final Host host, final boolean enabled) {
    +        final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
    +        if (outOfBandManagementConfig == null) {
    +            final OutOfBandManagementResponse response = new OutOfBandManagementResponse(null);
    +            response.setSuccess(false);
    +            response.setResultDescription("Out-of-band management is not configured for the host. Please configure the host before enabling/disabling it.");
    +            return response;
    +        }
    +        hostAlertCache.invalidate(host.getId());
    +        outOfBandManagementConfig.setEnabled(enabled);
    +        boolean updateResult = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
    +        if (updateResult && !enabled) {
    +            transitionPowerStateToDisabled(Collections.singletonList(host));
    +        }
    +        return buildEnableDisableResponse(enabled && updateResult);
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CONFIGURE, eventDescription = "updating out-of-band management configuration")
    +    public OutOfBandManagementResponse configureOutOfBandManagement(final Host host, final ImmutableMap<OutOfBandManagement.Option, String> options) {
    +        OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
    +        if (outOfBandManagementConfig == null) {
    +            outOfBandManagementConfig = outOfBandManagementDao.persist(new OutOfBandManagementVO(host.getId()));
    +        }
    +        outOfBandManagementConfig = updateConfig(outOfBandManagementConfig, options);
    +        if (Strings.isNullOrEmpty(outOfBandManagementConfig.getDriver()) || !outOfBandManagementDriversMap.containsKey(outOfBandManagementConfig.getDriver().toLowerCase())) {
    +            throw new CloudRuntimeException("Out-of-band management driver is not available. Please provide a valid driver name.");
    +        }
    +
    +        boolean updatedConfig = outOfBandManagementDao.update(outOfBandManagementConfig.getId(), (OutOfBandManagementVO) outOfBandManagementConfig);
    +        CallContext.current().setEventDetails("host id:" + host.getId() + " configuration:" + outOfBandManagementConfig.getAddress() + ":" + outOfBandManagementConfig.getPort());
    +
    +        if (!updatedConfig) {
    +            throw new CloudRuntimeException("Failed to update out-of-band management config for the host in the database.");
    +        }
    +
    +        String result = "Out-of-band management successfully configured for the host";
    +        LOG.debug(result);
    +
    +        final OutOfBandManagementResponse response = new OutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
    +        response.setResultDescription(result);
    +        response.setSuccess(true);
    +        return response;
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_ACTION, eventDescription = "issuing Host out-of-band management action", async = true)
    +    public OutOfBandManagementResponse executeOutOfBandManagementPowerOperation(final Host host, final OutOfBandManagement.PowerOperation powerOperation, final Long timeout) {
    +        checkOutOfBandManagementEnabledByZoneClusterHost(host);
    +        final OutOfBandManagement outOfBandManagementConfig = outOfBandManagementDao.findByHost(host.getId());
    +        final ImmutableMap<OutOfBandManagement.Option, String> options = getOptions(outOfBandManagementConfig);
    +        final OutOfBandManagementDriver driver = getDriver(outOfBandManagementConfig);
    +
    +        Long actionTimeOut = timeout;
    +        if (actionTimeOut == null) {
    +            actionTimeOut = OutOfBandManagementActionTimeout.valueIn(host.getClusterId());
    +        }
    +
    +        final OutOfBandManagementDriverPowerCommand cmd = new OutOfBandManagementDriverPowerCommand(options, actionTimeOut, powerOperation);
    +        final OutOfBandManagementDriverResponse driverResponse = driver.execute(cmd);
    +
    +        if (driverResponse == null) {
    +            throw new CloudRuntimeException(String.format("Out-of-band Management action (%s) on host (%s) failed due to no response from the driver", powerOperation, host.getUuid()));
    +        }
    +
    +        if (powerOperation.equals(OutOfBandManagement.PowerOperation.STATUS)) {
    +            transitionPowerState(driverResponse.toEvent(), outOfBandManagementConfig);
    +        }
    +
    +        if (!driverResponse.isSuccess()) {
    +            String errorMessage = String.format("Out-of-band Management action (%s) on host (%s) failed with error: %s", powerOperation, host.getUuid(), driverResponse.getError());
    +            if (driverResponse.hasAuthFailure()) {
    +                errorMessage = String.format("Out-of-band Management action (%s) on host (%s) failed due to authentication error: %s. Please check configured credentials.", powerOperation, host.getUuid(), driverResponse.getError());
    +                sendAuthError(host, errorMessage);
    +            }
    +            if (!powerOperation.equals(OutOfBandManagement.PowerOperation.STATUS)) {
    +                LOG.debug(errorMessage);
    +            }
    +            throw new CloudRuntimeException(errorMessage);
    +        }
    +
    +        final OutOfBandManagementResponse response = new OutOfBandManagementResponse(outOfBandManagementDao.findByHost(host.getId()));
    +        response.setSuccess(driverResponse.isSuccess());
    +        response.setResultDescription(driverResponse.getResult());
    +        response.setId(host.getUuid());
    +        response.setOutOfBandManagementAction(powerOperation.toString());
    +        return response;
    +    }
    +
    +    @Override
    +    @ActionEvent(eventType = EventTypes.EVENT_HOST_OUTOFBAND_MANAGEMENT_CHANGE_PASSWORD, eventDescription = "updating out-of-band management password")
    +    public OutOfBandManagementResponse changeOutOfBandManagementPassword(final Host host, final String newPassword) {
    +        checkOutOfBandManagementEnabledByZoneClusterHost(host);
    +        if (Strings.isNullOrEmpty(newPassword)) {
    +            throw new CloudRuntimeException(String.format("Cannot change out-of-band management password as provided new-password is null or empty for the host %s.", host.getUuid()));
    +        }
    +        GlobalLock outOfBandManagementHostLock = GlobalLock.getInternLock(getOutOfBandManagementHostLock(host.getId()));
    +        try {
    +            if (outOfBandManagementHostLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_HOST)) {
    +                try {
    --- End diff --
    
    we want to handle lock failures separately than critical code failure


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---