Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A2A74200CA7 for ; Tue, 30 May 2017 19:10:32 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A15DB160BB1; Tue, 30 May 2017 17:10:32 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2BE51160BDC for ; Tue, 30 May 2017 19:10:30 +0200 (CEST) Received: (qmail 42488 invoked by uid 500); 30 May 2017 17:10:24 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 41289 invoked by uid 99); 30 May 2017 17:10:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 30 May 2017 17:10:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 036B9DFC8B; Tue, 30 May 2017 17:10:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sunilg@apache.org To: common-commits@hadoop.apache.org Date: Tue, 30 May 2017 17:10:36 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He archived-at: Tue, 30 May 2017 17:10:32 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java deleted file mode 100644 index 9eb7d5c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.persist; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.io.Files; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.Path; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; -import org.apache.slider.common.params.Arguments; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.agent.AgentKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Class to prepare and persist app and add-on definitions. - * - * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit - * inclusion of application package in the config. - * - * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a - * temporary folder - upload to HDFS - * - * This class keeps track of all the required operations and allows them to be invoked by build operation - */ -public class AppDefinitionPersister { - private static final Logger log = - LoggerFactory.getLogger(AppDefinitionPersister.class); - - private final SliderFileSystem sliderFileSystem; - private List appDefinitions; - - public AppDefinitionPersister(SliderFileSystem sliderFileSystem) { - this.sliderFileSystem = sliderFileSystem; - appDefinitions = new ArrayList<>(); - } - - - /** - * Process the application package or folder by copying it to the cluster path - * - * @param appDefinition details of application package - * - * @throws BadConfigException - * @throws IOException - */ - private void persistDefinitionPackageOrFolder(AppDefinition appDefinition) - throws BadConfigException, IOException { - if (!appDefinition.appDefPkgOrFolder.canRead()) { - throw new BadConfigException("Pkg/Folder cannot be accessed - " - + appDefinition.appDefPkgOrFolder.getAbsolutePath()); - } - - File src = appDefinition.appDefPkgOrFolder; - String targetName = appDefinition.pkgName; - log.debug("Package name: " + targetName); - if (appDefinition.appDefPkgOrFolder.isDirectory()) { - log.info("Processing app package/folder {} for {}", - appDefinition.appDefPkgOrFolder.getAbsolutePath(), - appDefinition.pkgName); - File tmpDir = Files.createTempDir(); - File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName); - SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile); - src = zipFile; - } - - sliderFileSystem.getFileSystem().copyFromLocalFile( - false, - false, - new Path(src.toURI()), - new Path(appDefinition.targetFolderInFs, targetName)); - } - - public void persistPackages() throws BadConfigException, IOException { - for (AppDefinition appDefinition : appDefinitions) { - persistDefinitionPackageOrFolder(appDefinition); - } - } - - public void processSuppliedDefinitions(String clustername, - AbstractClusterBuildingActionArgs buildInfo, - ConfTreeOperations appConf) - throws BadConfigException, IOException, BadCommandArgumentsException { - // if metainfo is provided add to the app instance - if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) { - if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) { - throw new BadConfigException("Both %s and %s cannot be specified", - Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON); - } - - // Now we know that only one of either file or JSON is used - boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false; - String argUsed = isFileUsed ? Arguments.ARG_METAINFO - : Arguments.ARG_METAINFO_JSON; - - if (buildInfo.appDef != null) { - throw new BadConfigException("Both %s and %s cannot be specified", - argUsed, Arguments.ARG_APPDEF); - } - if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException( - "%s cannot not be set if %s is specified in the cmd line ", - AgentKeys.APP_DEF, argUsed); - } - - if (isFileUsed) { - if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) { - throw new BadConfigException( - "Path specified with %s either cannot be read or is not a file", - Arguments.ARG_METAINFO); - } - } else { - if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) { - throw new BadConfigException("Empty string specified with %s", - Arguments.ARG_METAINFO_JSON); - } - } - - File tempDir = Files.createTempDir(); - File pkgSrcDir = new File(tempDir, "default"); - if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) { - throw new IOException("Failed to create directory: " + pkgSrcDir); - } - File destMetaInfo = new File(pkgSrcDir, "metainfo.json"); - if (isFileUsed) { - if (buildInfo.appMetaInfo.getName().endsWith(".xml")) { - Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml")); - } else { - Files.copy(buildInfo.appMetaInfo, destMetaInfo); - } - } else { - Files.write( - buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")), - destMetaInfo); - } - - Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); - log.info("Using default app def path {}", appDirPath.toString()); - - appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, - SliderKeys.DEFAULT_APP_PKG)); - Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); - log.info("Setting app package to {}.", appDefPath); - } - - if (buildInfo.appDef != null) { - if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException("application.def must not be set if --appdef is provided."); - } - - if (!buildInfo.appDef.exists()) { - throw new BadConfigException("--appdef is not a valid path."); - } - - Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); - appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG)); - Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); - log.info("Setting app package to {}.", appDefPath); - } - - if (buildInfo.addonDelegate.getAddonMap().size() > 0) { - if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException("addon package can only be specified if main app package is specified."); - } - - List addons = new ArrayList(); - Map addonMap = buildInfo.addonDelegate.getAddonMap(); - for (Map.Entry entry : addonMap.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (SliderUtils.isUnset(value)) { - throw new BadConfigException("Invalid path for addon package " + key); - } - File defPath = new File(value); - if (!defPath.exists()) { - throw new BadConfigException("addon folder or package path is not valid."); - } - - Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); - String addonPkgName = "addon_" + key + ".zip"; - - log.debug( - "addonMap.get(key): {} addonPath: {} defPath: {} addonPkgName: {}", - addonMap.get(key), addonPath, defPath, addonPkgName); - - appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName)); - String addOnKey = AgentKeys.ADDON_PREFIX + key; - Path addonPkgPath = new Path(addonPath, addonPkgName); - log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath); - appConf.getGlobalOptions().set(addOnKey, addonPkgPath); - addons.add(addOnKey); - } - - String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); - if (SliderUtils.isUnset(existingList)) { - existingList = ""; - } - appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ",")); - } - } - - - @VisibleForTesting - public List getAppDefinitions() { - return appDefinitions; - } - - // Helper class to hold details for the app and addon packages - static class AppDefinition { - // The target folder where the package will be stored - public Path targetFolderInFs; - // The on disk location of the app def package or folder - public File appDefPkgOrFolder; - // Package name - public String pkgName; - - public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) { - this.targetFolderInFs = targetFolderInFs; - this.appDefPkgOrFolder = appDefPkgOrFolder; - this.pkgName = pkgName; - } - - @Override - public String toString() { - return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString()) - .append(", ") - .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString()) - .append(", ") - .append("pkgName").append(" : ").append(pkgName).toString(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java index 4f60c06..8fe2549 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java @@ -29,6 +29,7 @@ import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.codehaus.jackson.map.SerializationConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,11 @@ public class JsonSerDeser { mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); } + public JsonSerDeser(Class classType, PropertyNamingStrategy namingStrategy) { + this(classType); + mapper.setPropertyNamingStrategy(namingStrategy); + } + /** * Convert from JSON * @param json input http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index 01444fd..42e103a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -103,19 +103,6 @@ public abstract class AbstractClientProvider extends Configured { /** - * Any provider-side alteration of a configuration can take place here. - * @param aggregateConf config to patch - * @throws IOException IO problems - * @throws SliderException Slider-specific issues - */ - public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws - SliderException, - IOException { - //default: do nothing - } - - - /** * Prepare the AM settings for launch * @param fileSystem filesystem * @param serviceConf configuration of the client @@ -234,7 +221,7 @@ public abstract class AbstractClientProvider extends Configured { * @param appDescription brief description of the application * @return */ - public final Set createApplicationTags(String appName, + public static final Set createApplicationTags(String appName, String appVersion, String appDescription) { Set tags = new HashSet<>(); tags.add(SliderUtils.createNameTag(appName)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java deleted file mode 100644 index 41b26e9..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; -import org.apache.hadoop.registry.client.types.AddressTypes; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.main.ExitCodeProvider; -import org.apache.slider.server.appmaster.actions.QueueAccess; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.services.workflow.ForkedProcessService; -import org.apache.slider.server.services.workflow.ServiceParent; -import org.apache.slider.server.services.workflow.WorkflowSequenceService; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -/** - * The base class for provider services. It lets the implementations - * add sequences of operations, and propagates service failures - * upstream - */ -public abstract class AbstractProviderService - extends WorkflowSequenceService - implements - ProviderCore, - SliderKeys, - ProviderService { - private static final Logger log = - LoggerFactory.getLogger(AbstractProviderService.class); - protected StateAccessForProviders amState; - protected URL amWebAPI; - protected YarnRegistryViewForProviders yarnRegistry; - protected QueueAccess queueAccess; - - protected AbstractProviderService(String name) { - super(name); - setStopIfNoChildServicesAtStartup(false); - } - - @Override - public Configuration getConf() { - return getConfig(); - } - - public StateAccessForProviders getAmState() { - return amState; - } - - public QueueAccess getQueueAccess() { - return queueAccess; - } - - public void setAmState(StateAccessForProviders amState) { - this.amState = amState; - } - - @Override - public String getHumanName() { - return getName().toLowerCase(Locale.ENGLISH); - } - - @Override - public void bind(StateAccessForProviders stateAccessor, - QueueAccess queueAccess, - List liveContainers) { - this.amState = stateAccessor; - this.queueAccess = queueAccess; - } - - @Override - public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { - this.yarnRegistry = yarnRegistry; - } - - public YarnRegistryViewForProviders getYarnRegistry() { - return yarnRegistry; - } - - @Override - public void notifyContainerCompleted(ContainerId containerId) { - } - - /** - * Load default Configuration - * @param confDir configuration directory - * @return configuration - * @throws BadCommandArgumentsException - * @throws IOException - */ - @Override - public Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException { - return new Configuration(false); - } - - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @param siteXMLFilename provider-specific filename - * @return a configuration to be included in status - * @throws BadCommandArgumentsException argument problems - * @throws IOException IO problems - */ - protected Configuration loadProviderConfigurationInformation(File confDir, - String siteXMLFilename) - throws BadCommandArgumentsException, IOException { - Configuration siteConf; - File siteXML = new File(confDir, siteXMLFilename); - if (!siteXML.exists()) { - throw new BadCommandArgumentsException( - "Configuration directory %s doesn't contain %s - listing is %s", - confDir, siteXMLFilename, SliderUtils.listDir(confDir)); - } - - //now read it in - siteConf = ConfigHelper.loadConfFromFile(siteXML); - log.info("{} file is at {}", siteXMLFilename, siteXML); - log.info(ConfigHelper.dumpConfigToString(siteConf)); - return siteConf; - } - - /** - * No-op implementation of this method. - */ - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - } - - /** - * No-op implementation of this method. - * - * {@inheritDoc} - */ - @Override - public void validateApplicationConfiguration(AggregateConf instance, - File confDir, - boolean secure) - throws IOException, SliderException { - - } - - /** - * Scan through the roles and see if it is supported. - * @param role role to look for - * @return true if the role is known about -and therefore - * that a launcher thread can be deployed to launch it - */ - @Override - public boolean isSupportedRole(String role) { - Collection roles = getRoles(); - for (ProviderRole providedRole : roles) { - if (providedRole.name.equals(role)) { - return true; - } - } - return false; - } - - /** - * override point to allow a process to start executing in this container - * @param instanceDefinition cluster description - * @param confDir configuration directory - * @param env environment - * @param execInProgress the callback for the exec events - * @return false - * @throws IOException - * @throws SliderException - */ - @Override - public boolean exec(AggregateConf instanceDefinition, - File confDir, - Map env, - ProviderCompleted execInProgress) throws IOException, SliderException { - return false; - } - - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - @Override // ExitCodeProvider - public int getExitCode() { - Throwable cause = getFailureCause(); - if (cause != null) { - //failed for some reason - if (cause instanceof ExitCodeProvider) { - return ((ExitCodeProvider) cause).getExitCode(); - } - } - ForkedProcessService lastProc = latestProcess(); - if (lastProc == null || !lastProc.isProcessTerminated()) { - return 0; - } else { - return lastProc.getExitCode(); - } - } - - /** - * Return the latest forked process service that ran - * @return the forkes service - */ - protected ForkedProcessService latestProcess() { - Service current = getActiveService(); - Service prev = getPreviousService(); - - Service latest = current != null ? current : prev; - if (latest instanceof ForkedProcessService) { - return (ForkedProcessService) latest; - } else { - //its a composite object, so look inside it for a process - if (latest instanceof ServiceParent) { - return getFPSFromParentService((ServiceParent) latest); - } else { - //no match - return null; - } - } - } - - - /** - * Given a parent service, find the one that is a forked process - * @param serviceParent parent - * @return the forked process service or null if there is none - */ - protected ForkedProcessService getFPSFromParentService(ServiceParent serviceParent) { - List services = serviceParent.getServices(); - for (Service s : services) { - if (s instanceof ForkedProcessService) { - return (ForkedProcessService) s; - } - } - return null; - } - - /** - * if we are already running, start this service - */ - protected void maybeStartCommandSequence() { - if (isInState(STATE.STARTED)) { - startNextService(); - } - } - - /** - * Create a new forked process service with the given - * name, environment and command list -then add it as a child - * for execution in the sequence. - * - * @param name command name - * @param env environment - * @param commands command line - * @throws IOException - * @throws SliderException - */ - protected ForkedProcessService queueCommand(String name, - Map env, - List commands) throws - IOException, - SliderException { - ForkedProcessService process = buildProcess(name, env, commands); - //register the service for lifecycle management; when this service - //is terminated, so is the master process - addService(process); - return process; - } - - public ForkedProcessService buildProcess(String name, - Map env, - List commands) throws - IOException, - SliderException { - ForkedProcessService process; - process = new ForkedProcessService(name); - process.init(getConfig()); - process.build(env, commands); - return process; - } - - /* - * Build the provider status, can be empty - * @return the provider status - map of entries to add to the info section - */ - @Override - public Map buildProviderStatus() { - return new HashMap(); - } - - /* - Build the monitor details. The base implementation includes all the external URL endpoints - in the external view - */ - @Override - public Map buildMonitorDetails(ClusterDescription clusterDesc) { - Map details = new LinkedHashMap(); - - // add in all the endpoints - buildEndpointDetails(details); - - return details; - } - - @Override - public void buildEndpointDetails(Map details) { - ServiceRecord self = yarnRegistry.getSelfRegistration(); - - List externals = self.external; - for (Endpoint endpoint : externals) { - String addressType = endpoint.addressType; - if (AddressTypes.ADDRESS_URI.equals(addressType)) { - try { - List urls = RegistryTypeUtils.retrieveAddressURLs(endpoint); - if (!urls.isEmpty()) { - details.put(endpoint.api, new MonitorDetail(urls.get(0).toString(), true)); - } - } catch (InvalidRecordException | MalformedURLException ignored) { - // Ignored - } - - } - - } - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException { - this.amWebAPI = amWebURI; - } - - /** - * {@inheritDoc} - * - * - * @return The base implementation returns the most recent containers first. - */ - @Override - public ContainerReleaseSelector createContainerReleaseSelector() { - return new MostRecentContainerReleaseSelector(); - } - - @Override - public void releaseAssignedContainer(ContainerId containerId) { - // no-op - } - - @Override - public void addContainerRequest(AMRMClient.ContainerRequest req) { - // no-op - } - - @Override - public void cancelSingleRequest(AMRMClient.ContainerRequest request) { - // no-op - } - - @Override - public int cancelContainerRequests(Priority priority1, - Priority priority2, - int count) { - return 0; - } - - @Override - public void updateBlacklist(List blacklistAdditions, - List blacklistRemovals) { - // no-op - } - - @Override - public void execute(List operations) { - for (AbstractRMOperation operation : operations) { - operation.execute(this); - } - } - /** - * No-op implementation of this method. - */ - @Override - public void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles) { - } - - @Override - public boolean processContainerStatus(ContainerId containerId, - ContainerStatus status) { - return false; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java index 9767430..b07fc29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java @@ -19,9 +19,6 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.exceptions.SliderException; import java.util.List; public interface ProviderCore { @@ -31,13 +28,4 @@ public interface ProviderCore { List getRoles(); Configuration getConf(); - - /** - * Verify that an instance definition is considered valid by the provider - * @param instanceDefinition instance definition - * @throws SliderException if the configuration is not valid - */ - void validateInstanceDefinition(AggregateConf instanceDefinition) throws - SliderException; - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index 761ac0f..e0299e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -18,7 +18,7 @@ package org.apache.slider.providers; -import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Component; /** * Provider role and key for use in app requests. @@ -34,16 +34,8 @@ public final class ProviderRole { public int nodeFailureThreshold; public final long placementTimeoutSeconds; public final String labelExpression; + public final Component component; - public ProviderRole(String name, int id) { - this(name, - name, - id, - PlacementPolicy.DEFAULT, - ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS, - ResourceKeys.DEF_YARN_LABEL_EXPRESSION); - } /** * Create a provider role @@ -67,7 +59,7 @@ public final class ProviderRole { policy, nodeFailureThreshold, placementTimeoutSeconds, - labelExpression); + labelExpression, null); } /** @@ -87,7 +79,8 @@ public final class ProviderRole { int policy, int nodeFailureThreshold, long placementTimeoutSeconds, - String labelExpression) { + String labelExpression, + Component component) { this.name = name; if (group == null) { this.group = name; @@ -99,6 +92,8 @@ public final class ProviderRole { this.nodeFailureThreshold = nodeFailureThreshold; this.placementTimeoutSeconds = placementTimeoutSeconds; this.labelExpression = labelExpression; + this.component = component; + } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 4ca9326..c80de7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -18,18 +18,15 @@ package org.apache.slider.providers; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.slider.api.ClusterDescription; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.main.ExitCodeProvider; @@ -45,128 +42,17 @@ import java.net.URL; import java.util.List; import java.util.Map; -public interface ProviderService extends ProviderCore, - Service, - RMOperationHandlerActions, - ExitCodeProvider { +public interface ProviderService extends Service { /** * Set up the entire container launch context - * @param containerLauncher - * @param instanceDefinition - * @param container - * @param providerRole - * @param sliderFileSystem - * @param generatedConfPath - * @param appComponent - * @param containerTmpDirPath */ void buildContainerLaunchContext(ContainerLauncher containerLauncher, - AggregateConf instanceDefinition, - Container container, - ProviderRole providerRole, - SliderFileSystem sliderFileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws - IOException, - SliderException; + Application application, Container container, ProviderRole providerRole, + SliderFileSystem sliderFileSystem) throws IOException, SliderException; - /** - * Notify the providers of container completion - * @param containerId container that has completed - */ - void notifyContainerCompleted(ContainerId containerId); - - /** - * Execute a process in the AM - * @param instanceDefinition cluster description - * @param confDir configuration directory - * @param env environment - * @param execInProgress the callback for the exec events - * @return true if a process was actually started - * @throws IOException - * @throws SliderException - */ - boolean exec(AggregateConf instanceDefinition, - File confDir, - Map env, - ProviderCompleted execInProgress) throws IOException, - SliderException; - - /** - * Scan through the roles and see if it is supported. - * @param role role to look for - * @return true if the role is known about -and therefore - * that a launcher thread can be deployed to launch it - */ - boolean isSupportedRole(String role); - - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @return a configuration to be included in status - * @throws BadCommandArgumentsException - * @throws IOException - */ - Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException; - /** - * The application configuration should be initialized here - * - * @param instanceDefinition - * @param fileSystem - * @param roleGroup - * @throws IOException - * @throws SliderException - */ - void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, String roleGroup) throws IOException, - SliderException; - - /** - * This is a validation of the application configuration on the AM. - * Here is where things like the existence of keytabs and other - * not-seen-client-side properties can be tested, before - * the actual process is spawned. - * @param instanceDefinition clusterSpecification - * @param confDir configuration directory - * @param secure flag to indicate that secure mode checks must exist - * @throws IOException IO problemsn - * @throws SliderException any failure - */ - void validateApplicationConfiguration(AggregateConf instanceDefinition, - File confDir, - boolean secure - ) throws IOException, SliderException; - - /* - * Build the provider status, can be empty - * @return the provider status - map of entries to add to the info section - */ - Map buildProviderStatus(); - - /** - * Build a map of data intended for the AM webapp that is specific - * about this provider. The key is some text to be displayed, and the - * value can be a URL that will create an anchor over the key text. - * - * If no anchor is needed/desired, insert the key with a null value. - * @return the details - */ - Map buildMonitorDetails(ClusterDescription clusterSpec); - - /** - * Get a human friendly name for web UIs and messages - * @return a name string. Default is simply the service instance name. - */ - String getHumanName(); - - public void bind(StateAccessForProviders stateAccessor, - QueueAccess queueAccess, - List liveContainers); + void setAMState(StateAccessForProviders stateAccessForProviders); /** * Bind to the YARN registry @@ -175,39 +61,6 @@ public interface ProviderService extends ProviderCore, void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry); /** - * Build up the endpoint details for this service - * @param details - */ - void buildEndpointDetails(Map details); - - /** - * Prior to going live -register the initial service registry data - * @param amWebURI URL to the AM. This may be proxied, so use relative paths - * @param serviceRecord service record to build up - */ - void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException; - - /** - * Create the container release selector for this provider...any policy - * can be implemented - * @return the selector to use for choosing containers. - */ - ContainerReleaseSelector createContainerReleaseSelector(); - - /** - * On AM restart (for whatever reason) this API is required to rebuild the AM - * internal state with the containers which were already assigned and running - * - * @param liveContainers - * @param applicationId - * @param providerRoles - */ - void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles); - - /** * Process container status * @return true if status needs to be requested again, false otherwise */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/99c1074c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index f33db9b..f8ec976 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -18,7 +18,6 @@ package org.apache.slider.providers; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,24 +36,21 @@ import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.Configuration; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; -import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; @@ -66,16 +62,10 @@ import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.regex.Pattern; /** @@ -114,7 +104,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { */ public static boolean addProviderJar( Map providerResources, - Object provider, + Class providerClass, String jarName, SliderFileSystem sliderFileSystem, Path tempPath, @@ -125,7 +115,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { try { SliderUtils.putJar(providerResources, sliderFileSystem, - provider.getClass(), + providerClass, tempPath, libdir, jarName); @@ -138,32 +128,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } } } - - /** - * Add/overwrite the agent tarball (overwritten every time application is - * restarted). - * @param provider an instance of a provider class - * @param tarName name of the tarball to upload - * @param sliderFileSystem the file system - * @param agentDir directory to upload to - * @return true the location could be determined and the file added - * @throws IOException if the upload fails - */ - public static boolean addAgentTar(Object provider, - String tarName, - SliderFileSystem sliderFileSystem, - Path agentDir) throws - IOException { - File localFile = SliderUtils.findContainingJar(provider.getClass()); - if(localFile != null) { - String parentDir = localFile.getParent(); - Path agentTarPath = new Path(parentDir, tarName); - sliderFileSystem.getFileSystem().copyFromLocalFile(false, true, - agentTarPath, agentDir); - return true; - } - return false; - } /** * Loads all dependency jars from the default path. @@ -193,132 +157,24 @@ public class ProviderUtils implements RoleKeys, SliderKeys { libDir, libLocalSrcDir); } - - /** - * Validate the requested number of instances of a component. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param instanceDescription configuration - * @param name node class name - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(AggregateConf instanceDescription, - String name, int min, int max) - throws BadCommandArgumentsException { - MapOperations component = - instanceDescription.getResourceOperations().getComponent(name); - int count; - if (component == null) { - count = 0; - } else { - count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0); - } - validateNodeCount(name, count, min, max); - } - - /** - * Validate the count is between min and max. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param name node class name - * @param count requested node count - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(String name, - int count, - int min, - int max) throws BadCommandArgumentsException { - if (count < min) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is below the minimum of %d", name, count, - min); - } - if (max > 0 && count > max) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is above the maximum of %d", name, count, - max); - } - } - - /** - * Copy options beginning with "site.configName." prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param configName optional ".configName" portion of the prefix - * @param tokenMap key/value pairs to substitute into the option values - */ - public void propagateSiteOptions(Map options, - Map sitexml, - String configName, - Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX + - (!configName.isEmpty() ? configName + "." : ""); - propagateOptions(options, sitexml, tokenMap, prefix); - } - - /** - * Copy options beginning with prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param tokenMap key/value pairs to substitute into the option values - * @param prefix which options to copy to destination map - */ - public void propagateOptions(Map options, - Map sitexml, - Map tokenMap, - String prefix) { - for (Map.Entry entry : options.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String envName = key.substring(prefix.length()); - if (!envName.isEmpty()) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } - } - sitexml.put(envName, value); - } - } - } - } - - /** - * Substitute tokens into option map values, returning a new map. - * @param options source map - * @param tokenMap key/value pairs to substitute into the option values - * @return map with substituted values - */ - public Map filterSiteOptions(Map options, + // Build key -> value map + // value will be substituted by corresponding data in tokenMap + public Map substituteConfigs(Map configs, Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX; String format = "${%s}"; Map filteredOptions = new HashMap<>(); - for (Map.Entry entry : options.entrySet()) { + for (Map.Entry entry : configs.entrySet()) { String key = entry.getKey(); - if (key.startsWith(prefix)) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } + String value = entry.getValue(); + if (tokenMap != null) { + for (Map.Entry token : tokenMap.entrySet()) { + value = + value.replaceAll(Pattern.quote(token.getKey()), token.getValue()); } - filteredOptions.put(String.format(format, key), value); } + filteredOptions.put(String.format(format, key), value); } + return filteredOptions; } @@ -345,28 +201,27 @@ public class ProviderUtils implements RoleKeys, SliderKeys { return intVal; } + /** * Localize the service keytabs for the application. * @param launcher container launcher - * @param instanceDefinition app specification * @param fileSystem file system - * @param clusterName app name * @throws IOException trouble uploading to HDFS */ public void localizeServiceKeytabs(ContainerLauncher launcher, - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String clusterName) throws IOException { - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - String keytabPathOnHost = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + SliderFileSystem fileSystem, Application application) throws IOException { + + Configuration conf = application.getConfiguration(); + String keytabPathOnHost = + conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + String amKeytabName = + conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = + conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); // we need to localize the keytab files in the directory Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - clusterName); + application.getName()); boolean serviceKeytabsDeployed = false; if (fileSystem.getFileSystem().exists(keytabDirPath)) { FileStatus[] keytabs = fileSystem.getFileSystem().listStatus( @@ -395,591 +250,119 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } - /** - * Upload a local file to the cluster security dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadSecurityResource(File resource, - SliderFileSystem fileSystem, String clusterName) throws IOException { - Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName); - return uploadResource(resource, fileSystem, certsDir); - } - - /** - * Upload a local file to the cluster resources dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param roleName optional subdirectory (for component-specific resources) - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadResource(File resource, SliderFileSystem fileSystem, - String roleName, String clusterName) throws IOException { - Path dir; - if (roleName == null) { - dir = fileSystem.buildClusterResourcePath(clusterName); - } else { - dir = fileSystem.buildClusterResourcePath(clusterName, roleName); - } - return uploadResource(resource, fileSystem, dir); - } - - /** - * Upload a local file to a specified HDFS directory. If the file already - * exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param parentDir destination directory in HDFS - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private synchronized Path uploadResource(File resource, - SliderFileSystem fileSystem, Path parentDir) throws IOException { - if (!fileSystem.getFileSystem().exists(parentDir)) { - fileSystem.getFileSystem().mkdirs(parentDir, + // 1. Create all config files for a component on hdfs for localization + // 2. Add the config file to localResource + //TODO handle Template format config file + public void createConfigFileAndAddLocalResource(ContainerLauncher launcher, + SliderFileSystem fs, Component component, + Map tokensForSubstitution, + StateAccessForProviders amState) throws IOException { + Path compDir = + new Path(new Path(fs.getAppDir(), "components"), component.getName()); + if (!fs.getFileSystem().exists(compDir)) { + fs.getFileSystem().mkdirs(compDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - } - Path destPath = new Path(parentDir, resource.getName()); - if (!fileSystem.getFileSystem().exists(destPath)) { - FSDataOutputStream os = null; - try { - os = fileSystem.getFileSystem().create(destPath); - byte[] contents = FileUtils.readFileToByteArray(resource); - os.write(contents, 0, contents.length); - os.flush(); - } finally { - IOUtils.closeStream(os); - } - log.info("Uploaded {} to localization path {}", resource, destPath); + log.info("Creating component dir: " + compDir); } else { - log.info("Resource {} already existed at localization path {}", resource, - destPath); - } - - while (!fileSystem.getFileSystem().exists(destPath)) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - // ignore - } - } - - fileSystem.getFileSystem().setPermission(destPath, - new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); - - return destPath; - } - - /** - * Write a configuration property map to a local file in a specified format. - * @param fileSystem file system - * @param file destination file - * @param configFormat file format - * @param configFileDN file description - * @param config properties to save to the file - * @param clusterName app name - * @throws IOException file cannot be created - */ - private synchronized void createConfigFile(SliderFileSystem fileSystem, - File file, ConfigFormat configFormat, String configFileDN, - Map config, String clusterName) throws IOException { - if (file.exists()) { - log.info("Skipping writing {} file {} because it already exists", - configFormat, file); - return; - } - log.info("Writing {} file {}", configFormat, file); - - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, clusterName, file.getName()); - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(configFileDN, - config.entrySet()); - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(configFormat, - publishedConfiguration); - configurationOutputter.save(file); - } - - /** - * Determine config files requested in the appConf, generate the files, and - * localize them. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param appConf app configurations - * @param configs configurations grouped by config name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @throws IOException file(s) cannot be uploaded - * @throws BadConfigException file name not specified or file format not - * supported - */ - public void localizeConfigFiles(ContainerLauncher launcher, - String roleName, String roleGroup, - ConfTreeOperations appConf, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName) - throws IOException, BadConfigException { - for (Entry> configEntry : configs.entrySet()) { - String configFileName = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .NAME_SUFFIX, null); - String configFileType = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .TYPE_SUFFIX, null); - if (configFileName == null && configFileType == null) { - // config file not requested, so continue - continue; - } - if (configFileName == null) { - throw new BadConfigException("Config file name null for " + - configEntry.getKey()); - } - if (configFileType == null) { - throw new BadConfigException("Config file type null for " + - configEntry.getKey()); - } - ConfigFormat configFormat = ConfigFormat.resolve(configFileType); - if (configFormat == null) { - throw new BadConfigException("Config format " + configFileType + - " doesn't exist"); - } - boolean perComponent = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_COMPONENT, false); - boolean perGroup = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_GROUP, false); - - localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(), - configFormat, configFileName, configs, env, fileSystem, - clusterName, perComponent, perGroup); - } - } - - /** - * Create and localize a config file. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param configFileDN config description/name - * @param configFormat config format - * @param configFileName config file name - * @param configs configs grouped by config description/name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @param perComponent true if file should be created per unique component - * @param perGroup true if file should be created per component group - * @throws IOException file cannot be uploaded - */ - public void localizeConfigFile(ContainerLauncher launcher, - String roleName, String roleGroup, - String configFileDN, ConfigFormat configFormat, String configFileName, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName, - boolean perComponent, - boolean perGroup) - throws IOException { - if (launcher == null) { + log.info("Component conf dir already exists: " + compDir); return; } - Map config = ConfigUtils.replacePropsInConfig( - configs.get(configFileDN), env.options); - String fileName = ConfigUtils.replaceProps(config, configFileName); - File localFile = new File(RESOURCE_DIR); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(RESOURCE_DIR + " could not be created!"); - } - } - - String folder = null; - if (perComponent) { - folder = roleName; - } else if (perGroup) { - folder = roleGroup; - } - if (folder != null) { - localFile = new File(localFile, folder); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(localFile + " could not be created!"); - } - } - } - localFile = new File(localFile, new File(fileName).getName()); - - log.info("Localizing {} configs to config file {} (destination {}) " + - "based on {} configs", config.size(), localFile, fileName, - configFileDN); - if (!localFile.exists()) { - createConfigFile(fileSystem, localFile, configFormat, configFileDN, - config, clusterName); - } else { - log.info("Local {} file {} already exists", configFormat, localFile); - } - Path destPath = uploadResource(localFile, fileSystem, folder, clusterName); - LocalResource configResource = fileSystem.createAmResource(destPath, - LocalResourceType.FILE); - - File destFile = new File(fileName); - if (destFile.isAbsolute()) { - launcher.addLocalResource( - RESOURCE_DIR + "/" + destFile.getName(), - configResource, fileName); - } else { - launcher.addLocalResource(APP_CONF_DIR + "/" + fileName, - configResource); - } - } - /** - * Localize application tarballs and other resources requested by the app. - * @param launcher container launcher - * @param fileSystem file system - * @param appConf app configurations - * @param roleGroup component group - * @param clusterName app name - * @throws IOException resources cannot be uploaded - * @throws BadConfigException package name or type is not specified - */ - public void localizePackages(ContainerLauncher launcher, - SliderFileSystem fileSystem, ConfTreeOperations appConf, String roleGroup, - String clusterName) throws IOException, BadConfigException { - for (Entry> pkg : - getPackages(roleGroup, appConf).entrySet()) { - String pkgName = pkg.getValue().get(OptionKeys.NAME_SUFFIX); - String pkgType = pkg.getValue().get(OptionKeys.TYPE_SUFFIX); - Path pkgPath = fileSystem.buildResourcePath(pkgName); - if (!fileSystem.isFile(pkgPath)) { - pkgPath = fileSystem.buildResourcePath(clusterName, - pkgName); - } - if (!fileSystem.isFile(pkgPath)) { - throw new IOException("Package doesn't exist as a resource: " + - pkgName); + for (ConfigFile configFile : component.getConfiguration().getFiles()) { + String fileName = configFile.getSrcFile(); + // substitute file name + for (Map.Entry token : tokensForSubstitution.entrySet()) { + configFile.setDestFile(configFile.getDestFile() + .replaceAll(Pattern.quote(token.getKey()), token.getValue())); } - log.info("Adding resource {}", pkgName); - LocalResourceType type = LocalResourceType.FILE; - if ("archive".equals(pkgType)) { - type = LocalResourceType.ARCHIVE; - } - LocalResource packageResource = fileSystem.createAmResource( - pkgPath, type); - launcher.addLocalResource(APP_PACKAGES_DIR, packageResource); - } - } - - /** - * Build a map of configuration description/name to configuration key/value - * properties, with all known tokens substituted into the property values. - * @param appConf app configurations - * @param internalsConf internal configurations - * @param containerId container ID - * @param roleName component name - * @param roleGroup component group - * @param amState access to AM state - * @return configuration properties grouped by config description/name - */ - public Map> buildConfigurations( - ConfTreeOperations appConf, ConfTreeOperations internalsConf, - String containerId, String clusterName, String roleName, String roleGroup, - StateAccessForProviders amState) { - - Map> configurations = new TreeMap<>(); - Map tokens = getStandardTokenMap(appConf, - internalsConf, roleName, roleGroup, containerId, clusterName); - - Set configs = new HashSet<>(); - configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf)); - configs.addAll(getSystemConfigurationsRequested(appConf)); - - for (String configType : configs) { - addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, amState); - if (appConf.getComponent(roleGroup) != null) { - addNamedConfiguration(configType, - appConf.getComponent(roleGroup).options, configurations, tokens, - amState); - } - } - - //do a final replacement of re-used configs - dereferenceAllConfigs(configurations); - - return configurations; - } - - /** - * Substitute "site." prefixed configuration values into other configuration - * values where needed. The format for these substitutions is that - * {@literal ${@//site/configDN/key}} will be replaced by the value for the - * "site.configDN.key" property. - * @param configurations configuration properties grouped by config - * description/name - */ - public void dereferenceAllConfigs( - Map> configurations) { - Map allConfigs = new HashMap<>(); - String lookupFormat = "${@//site/%s/%s}"; - for (Map.Entry> entry : configurations.entrySet()) { - Map configBucket = entry.getValue(); - for(Map.Entry config: configBucket.entrySet()) { - allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()), - config.getValue()); - } - } - - boolean finished = false; - while (!finished) { - finished = true; - for (Map.Entry entry : allConfigs.entrySet()) { - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); - } - } - if (configValue != null && !configValue.equals(entry.getValue())) { - finished = false; - allConfigs.put(entry.getKey(), configValue); - } - } - } - for (Map.Entry> configEntry : configurations - .entrySet()) { - Map configBucket = configEntry.getValue(); - for (Map.Entry entry: configBucket.entrySet()) { - String configName = entry.getKey(); - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); + // substitute configs + substituteConfigs(configFile.getProps(), tokensForSubstitution); + + // write configs onto hdfs + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(fileName, + configFile.getProps().entrySet()); + Path remoteFile = new Path(compDir, fileName); + if (!fs.getFileSystem().exists(remoteFile)) { + synchronized (this) { + if (!fs.getFileSystem().exists(remoteFile)) { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter( + ConfigFormat.resolve(configFile.getType().toString()), + publishedConfiguration); + FSDataOutputStream os = null; + try { + os = fs.getFileSystem().create(remoteFile); + configurationOutputter.save(os); + os.flush(); + log.info("Created config file on hdfs: " + remoteFile); + } finally { + IOUtils.closeStream(os); + } } } - configBucket.put(configName, configValue); - } - } - } - - /** - * Return a set of configuration description/names represented in the app. - * configuration - * @param roleGroup component group - * @param appConf app configurations - * @return set of configuration description/names - */ - public Set getApplicationConfigurationTypes(String roleGroup, - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - - String prefix = OptionKeys.CONF_FILE_PREFIX; - String suffix = OptionKeys.TYPE_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component != null) { - addConfsToList(component, configList, prefix, suffix); - } - addConfsToList(appConf.getGlobalOptions(), configList, prefix, suffix); - - return configList; - } - - /** - * Finds all configuration description/names of the form - * prefixconfigDNsuffix in the configuration (e.g. conf.configDN.type). - * @param confMap configuration properties - * @param confList set containing configuration description/names - * @param prefix configuration key prefix to match - * @param suffix configuration key suffix to match - */ - private void addConfsToList(Map confMap, - Set confList, String prefix, String suffix) { - for (Entry entry : confMap.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix) && key.endsWith(suffix)) { - String confName = key.substring(prefix.length(), - key.length() - suffix.length()); - if (!confName.isEmpty()) { - confList.add(confName); - } - } - } - } - - /** - * Build a map of package description/name to package key/value properties - * (there should be two properties, type and name). - * @param roleGroup component group - * @param appConf app configurations - * @return map of package description/name to package key/value properties - * @throws BadConfigException package name or type is not specified - */ - public Map> getPackages(String roleGroup, - ConfTreeOperations appConf) throws BadConfigException { - Map> packages = new HashMap<>(); - String prefix = OptionKeys.PKG_FILE_PREFIX; - String typeSuffix = OptionKeys.TYPE_SUFFIX; - String nameSuffix = OptionKeys.NAME_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component == null) { - component = appConf.getGlobalOptions(); - } - for (Map.Entry entry : component.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String confName; - String type; - if (key.endsWith(typeSuffix)) { - confName = key.substring(prefix.length(), key.length() - typeSuffix.length()); - type = typeSuffix; - } else if (key.endsWith(nameSuffix)) { - confName = key.substring(prefix.length(), key.length() - nameSuffix.length()); - type = nameSuffix; - } else { - continue; - } - if (!packages.containsKey(confName)) { - packages.put(confName, new HashMap()); - } - packages.get(confName).put(type, entry.getValue()); - } - } - - for (Entry> pkg : packages.entrySet()) { - if (!pkg.getValue().containsKey(OptionKeys.TYPE_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package type"); } - if (!pkg.getValue().containsKey(OptionKeys.NAME_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package name"); - } - } - - return packages; - } - - /** - * Return system configurations requested by the app. - * @param appConf app configurations - * @return set of system configurations - */ - public Set getSystemConfigurationsRequested( - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - String configTypes = appConf.get(SYSTEM_CONFIGS); - if (configTypes != null && configTypes.length() > 0) { - String[] configs = configTypes.split(","); - for (String config : configs) { - configList.add(config.trim()); + // Publish configs + amState.getPublishedSliderConfigurations() + .put(configFile.getSrcFile(), publishedConfiguration); + + // Add resource for localization + LocalResource configResource = + fs.createAmResource(remoteFile, LocalResourceType.FILE); + File destFile = new File(configFile.getDestFile()); + //TODO why to we need to differetiate RESOURCE_DIR vs APP_CONF_DIR + if (destFile.isAbsolute()) { + String symlink = RESOURCE_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource, + configFile.getDestFile()); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile() + ", dest mount path: " + + configFile.getDestFile()); + } else { + String symlink = APP_CONF_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile()); } } - - return configList; - } - - /** - * For a given config description/name, pull out its site configs from the - * source config map, remove the site.configDN. prefix from them, and place - * them into a new config map using the {@link #propagateSiteOptions} method - * (with tokens substituted). This new k/v map is put as the value for the - * configDN key in the configurations map. - * @param configName config description/name - * @param sourceConfig config containing site.* properties - * @param configurations configuration map to be populated - * @param tokens initial substitution tokens - * @param amState access to AM state - */ - private void addNamedConfiguration(String configName, - Map sourceConfig, - Map> configurations, - Map tokens, StateAccessForProviders amState) { - Map config = new HashMap<>(); - if (configName.equals(GLOBAL_CONFIG_TAG)) { - addDefaultGlobalConfig(config); - } - // add role hosts to tokens - addRoleRelatedTokens(tokens, amState); - propagateSiteOptions(sourceConfig, config, configName, tokens); - - configurations.put(configName, config); } /** * Get initial token map to be substituted into config values. * @param appConf app configurations - * @param internals internal configurations - * @param componentName component name - * @param componentGroup component group - * @param clusterName app name - * @return tokens to replace - */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, - String componentGroup, String clusterName) { - return getStandardTokenMap(appConf, internals, componentName, - componentGroup, null, clusterName); - } - - /** - * Get initial token map to be substituted into config values. - * @param appConf app configurations - * @param internals internal configurations * @param componentName component name * @param componentGroup component group * @param containerId container ID * @param clusterName app name * @return tokens to replace */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, + public Map getStandardTokenMap( + Configuration appConf, Configuration componentConf, String componentName, String componentGroup, String containerId, String clusterName) { Map tokens = new HashMap<>(); if (containerId != null) { tokens.put("${CONTAINER_ID}", containerId); } - String nnuri = appConf.get("site.fs.defaultFS"); - tokens.put("${NN_URI}", nnuri); - tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); - tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); - tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); - String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX, - null); + String nnuri = appConf.getProperty("fs.defaultFS"); + if (nnuri != null && !nnuri.isEmpty()) { + tokens.put("${NN_URI}", nnuri); + tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); + } + tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS)); + tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH)); + String prefix = componentConf.getProperty(ROLE_PREFIX); String dataDirSuffix = ""; if (prefix == null) { prefix = ""; } else { dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix); } - tokens.put("${DEFAULT_DATA_DIR}", internals.getGlobalOptions() - .getOption(InternalKeys.INTERNAL_DATA_DIR_PATH, null) + dataDirSuffix); - tokens.put("${JAVA_HOME}", appConf.get(JAVA_HOME)); + tokens.put("${DEFAULT_DATA_DIR}", + appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH) + + dataDirSuffix); + tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); tokens.put("${COMPONENT_PREFIX}", prefix); @@ -1005,7 +388,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * @param tokens existing tokens * @param amState access to AM state */ - public void addRoleRelatedTokens(Map tokens, + public void addRoleHostTokens(Map tokens, StateAccessForProviders amState) { if (amState == null) { return; @@ -1020,26 +403,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } /** - * Add global configuration properties. - * @param config map where default global properties will be added - */ - private void addDefaultGlobalConfig(Map config) { - config.put("app_log_dir", "${LOG_DIR}"); - config.put("app_pid_dir", "${WORK_DIR}/app/run"); - config.put("app_install_dir", "${WORK_DIR}/app/install"); - config.put("app_conf_dir", "${WORK_DIR}/" + APP_CONF_DIR); - config.put("app_input_conf_dir", "${WORK_DIR}/" + PROPAGATED_CONF_DIR_NAME); - - // add optional parameters only if they are not already provided - if (!config.containsKey("pid_file")) { - config.put("pid_file", "${WORK_DIR}/app/run/component.pid"); - } - if (!config.containsKey("app_root")) { - config.put("app_root", "${WORK_DIR}/app/install"); - } - } - - /** * Return a list of hosts based on current ClusterNodes. * @param values cluster nodes * @param hostOnly whether host or host/server name will be added to list @@ -1101,82 +464,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys { containerId, e); } } - - /** - * Publish a named property bag that may contain name-value pairs for app - * configurations such as hbase-site. - * @param name config file identifying name - * @param description config file description - * @param entries config file properties - * @param amState access to AM state - */ - public void publishApplicationInstanceData(String name, String description, - Iterable> entries, - StateAccessForProviders amState) { - PublishedConfiguration pubconf = new PublishedConfiguration(description, - entries); - log.info("publishing {}", pubconf); - amState.getPublishedSliderConfigurations().put(name, pubconf); - } - - /** - * Publish an export group. - * @param exportGroup export groups - * @param amState access to AM state - * @param groupName export group name - */ - public void publishExportGroup( - Map> exportGroup, - StateAccessForProviders amState, String groupName) { - // Publish in old format for the time being - Map simpleEntries = new HashMap<>(); - for (Entry> entry : exportGroup.entrySet()) { - Set exports = entry.getValue(); - if (SliderUtils.isNotEmpty(exports)) { - Set values = new TreeSet<>(); - for (ExportEntry export : exports) { - values.add(export.getValue()); - } - simpleEntries.put(entry.getKey(), StringUtils.join(",", values)); - } - } - publishApplicationInstanceData(groupName, groupName, - simpleEntries.entrySet(), amState); - - PublishedExports exports = new PublishedExports(groupName); - exports.setUpdated(new Date().getTime()); - exports.putValues(exportGroup.entrySet()); - amState.getPublishedExportsSet().put(groupName, exports); - } - - public Map getExports(ConfTreeOperations appConf, - String roleGroup) { - Map exports = new HashMap<>(); - propagateOptions(appConf.getComponent(roleGroup).options, exports, - null, OptionKeys.EXPORT_PREFIX); - return exports; - } - - public String getGroupKey(String roleGroup, ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleGroup); - } - - public String getNameKey(String roleName, String roleGroup, - ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleName); - } - - public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) { - if (!rolePrefix.isEmpty()) { - if (!roleNameOrGroup.startsWith(rolePrefix)) { - log.warn("Something went wrong, {} doesn't start with {}", - roleNameOrGroup, rolePrefix); - return null; - } - roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length()); - } - return roleNameOrGroup.toUpperCase(Locale.ENGLISH); - } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org