Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 36385200BD1 for ; Mon, 28 Nov 2016 22:16:10 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 351DE160B26; Mon, 28 Nov 2016 21:16:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 88462160B0D for ; Mon, 28 Nov 2016 22:16:07 +0100 (CET) Received: (qmail 49622 invoked by uid 500); 28 Nov 2016 21:15:53 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 47178 invoked by uid 99); 28 Nov 2016 21:15:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 28 Nov 2016 21:15:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 18CFDF17E3; Mon, 28 Nov 2016 21:15:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Mon, 28 Nov 2016 21:16:26 -0000 Message-Id: <8bd174d0710c44338b81a2e9d7876213@git.apache.org> In-Reply-To: <3022f5161bb4468c92a2825216e9111f@git.apache.org> References: <3022f5161bb4468c92a2825216e9111f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [38/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe) archived-at: Mon, 28 Nov 2016 21:16:10 -0000 YARN-5461. Initial code ported from slider-core module. (jianhe) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a379904d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a379904d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a379904d Branch: refs/heads/yarn-native-services Commit: a379904d28836cd9247b2242cb1e287f8b789f8a Parents: 5d5614f Author: Jian He Authored: Wed Aug 3 00:18:01 2016 +0800 Committer: Jian He Committed: Mon Nov 28 13:09:08 2016 -0800 ---------------------------------------------------------------------- .../hadoop-yarn-slider-core/pom.xml | 397 + .../src/assembly/executable-jar.xml | 47 + .../src/license/THIRD-PARTY.properties | 33 + .../slider/server/avro/RoleHistoryRecord.avsc | 114 + .../apache/hadoop/security/KerberosDiags.java | 676 + .../src/main/java/org/apache/slider/Slider.java | 52 + .../apache/slider/api/ClusterDescription.java | 795 + .../slider/api/ClusterDescriptionKeys.java | 25 + .../api/ClusterDescriptionOperations.java | 93 + .../java/org/apache/slider/api/ClusterNode.java | 220 + .../org/apache/slider/api/InternalKeys.java | 199 + .../java/org/apache/slider/api/OptionKeys.java | 58 + .../org/apache/slider/api/ResourceKeys.java | 201 + .../java/org/apache/slider/api/RoleKeys.java | 116 + .../apache/slider/api/SliderApplicationApi.java | 167 + .../slider/api/SliderClusterProtocol.java | 179 + .../java/org/apache/slider/api/StateValues.java | 53 + .../java/org/apache/slider/api/StatusKeys.java | 117 + .../org/apache/slider/api/proto/Messages.java | 34473 +++++++++++++++++ .../slider/api/proto/RestTypeMarshalling.java | 319 + .../slider/api/proto/SliderClusterAPI.java | 2293 ++ .../types/ApplicationLivenessInformation.java | 53 + .../slider/api/types/ComponentInformation.java | 107 + .../slider/api/types/ContainerInformation.java | 58 + .../slider/api/types/NodeEntryInformation.java | 78 + .../slider/api/types/NodeInformation.java | 59 + .../slider/api/types/NodeInformationList.java | 41 + .../slider/api/types/PingInformation.java | 47 + .../apache/slider/api/types/RoleStatistics.java | 66 + .../api/types/SliderInstanceDescription.java | 54 + .../slider/client/ClientRegistryBinder.java | 201 + .../org/apache/slider/client/ClientUtils.java | 111 + .../org/apache/slider/client/SliderClient.java | 4569 +++ .../apache/slider/client/SliderClientAPI.java | 368 + .../slider/client/SliderYarnClientImpl.java | 410 + .../apache/slider/client/TokensOperation.java | 109 + .../client/ipc/SliderApplicationIpcClient.java | 245 + .../client/ipc/SliderClusterOperations.java | 529 + .../slider/client/rest/BaseRestClient.java | 152 + .../slider/client/rest/RestClientFactory.java | 89 + .../rest/SliderApplicationApiRestClient.java | 326 + .../org/apache/slider/common/Constants.java | 35 + .../apache/slider/common/SliderExitCodes.java | 88 + .../org/apache/slider/common/SliderKeys.java | 278 + .../common/SliderXMLConfKeysForTesting.java | 83 + .../apache/slider/common/SliderXmlConfKeys.java | 206 + .../common/params/AbstractActionArgs.java | 178 + .../common/params/AbstractArgsDelegate.java | 26 + .../AbstractClusterBuildingActionArgs.java | 217 + .../common/params/ActionAMSuicideArgs.java | 44 + .../slider/common/params/ActionBuildArgs.java | 32 + .../slider/common/params/ActionClientArgs.java | 98 + .../slider/common/params/ActionCreateArgs.java | 59 + .../common/params/ActionDependencyArgs.java | 65 + .../slider/common/params/ActionDestroyArgs.java | 37 + .../common/params/ActionDiagnosticArgs.java | 73 + .../slider/common/params/ActionEchoArgs.java | 33 + .../slider/common/params/ActionExistsArgs.java | 47 + .../slider/common/params/ActionFlexArgs.java | 54 + .../slider/common/params/ActionFreezeArgs.java | 56 + .../slider/common/params/ActionHelpArgs.java | 51 + .../common/params/ActionInstallKeytabArgs.java | 57 + .../common/params/ActionInstallPackageArgs.java | 58 + .../slider/common/params/ActionKDiagArgs.java | 86 + .../slider/common/params/ActionKeytabArgs.java | 81 + .../common/params/ActionKillContainerArgs.java | 37 + .../slider/common/params/ActionListArgs.java | 74 + .../slider/common/params/ActionLookupArgs.java | 76 + .../slider/common/params/ActionNodesArgs.java | 71 + .../slider/common/params/ActionPackageArgs.java | 81 + .../common/params/ActionRegistryArgs.java | 218 + .../slider/common/params/ActionResolveArgs.java | 153 + .../common/params/ActionResourceArgs.java | 68 + .../slider/common/params/ActionStatusArgs.java | 45 + .../slider/common/params/ActionThawArgs.java | 61 + .../slider/common/params/ActionTokensArgs.java | 78 + .../slider/common/params/ActionUpdateArgs.java | 32 + .../slider/common/params/ActionUpgradeArgs.java | 73 + .../slider/common/params/ActionVersionArgs.java | 46 + .../slider/common/params/AddonArgsDelegate.java | 54 + .../params/AppAndResouceOptionArgsDelegate.java | 111 + .../org/apache/slider/common/params/ArgOps.java | 157 + .../apache/slider/common/params/Arguments.java | 162 + .../apache/slider/common/params/ClientArgs.java | 385 + .../apache/slider/common/params/CommonArgs.java | 303 + .../common/params/ComponentArgsDelegate.java | 52 + .../common/params/DontSplitArguments.java | 34 + .../common/params/LaunchArgsAccessor.java | 30 + .../common/params/LaunchArgsDelegate.java | 51 + .../common/params/PathArgumentConverter.java | 34 + .../slider/common/params/SliderAMArgs.java | 57 + .../common/params/SliderAMCreateAction.java | 74 + .../slider/common/params/SliderActions.java | 113 + .../common/params/URIArgumentConverter.java | 40 + .../common/params/URLArgumentConverter.java | 40 + .../slider/common/params/WaitArgsDelegate.java | 42 + .../slider/common/params/WaitTimeAccessor.java | 24 + .../apache/slider/common/tools/Comparators.java | 69 + .../slider/common/tools/ConfigHelper.java | 658 + .../slider/common/tools/CoreFileSystem.java | 915 + .../apache/slider/common/tools/Duration.java | 109 + .../apache/slider/common/tools/PortScanner.java | 113 + .../slider/common/tools/SliderFileSystem.java | 42 + .../apache/slider/common/tools/SliderUtils.java | 2548 ++ .../slider/common/tools/SliderVersionInfo.java | 108 + .../conf/AbstractInputPropertiesValidator.java | 49 + .../apache/slider/core/conf/AggregateConf.java | 198 + .../org/apache/slider/core/conf/ConfTree.java | 114 + .../slider/core/conf/ConfTreeOperations.java | 477 + .../core/conf/InputPropertiesValidator.java | 27 + .../apache/slider/core/conf/MapOperations.java | 344 + .../conf/ResourcesInputPropertiesValidator.java | 41 + .../conf/TemplateInputPropertiesValidator.java | 38 + .../exceptions/BadClusterStateException.java | 35 + .../BadCommandArgumentsException.java | 30 + .../core/exceptions/BadConfigException.java | 39 + .../slider/core/exceptions/ErrorStrings.java | 57 + .../core/exceptions/ExceptionConverter.java | 128 + .../core/exceptions/NoSuchNodeException.java | 32 + .../core/exceptions/NotFoundException.java | 35 + .../exceptions/ServiceNotReadyException.java | 43 + .../slider/core/exceptions/SliderException.java | 67 + .../SliderInternalStateException.java | 34 + .../TriggerClusterTeardownException.java | 41 + .../UnknownApplicationInstanceException.java | 51 + .../slider/core/exceptions/UsageException.java | 34 + .../core/exceptions/WaitTimeoutException.java | 34 + .../slider/core/launch/AbstractLauncher.java | 528 + .../slider/core/launch/AppMasterLauncher.java | 233 + .../core/launch/ClasspathConstructor.java | 172 + .../slider/core/launch/CommandLineBuilder.java | 104 + .../slider/core/launch/ContainerLauncher.java | 72 + .../slider/core/launch/CredentialUtils.java | 379 + .../core/launch/JavaCommandLineBuilder.java | 182 + .../slider/core/launch/LaunchedApplication.java | 108 + .../slider/core/launch/RunningApplication.java | 76 + .../launch/SerializedApplicationReport.java | 98 + .../slider/core/main/ExitCodeProvider.java | 32 + .../org/apache/slider/core/main/IrqHandler.java | 103 + .../slider/core/main/LauncherExitCodes.java | 196 + .../org/apache/slider/core/main/RunService.java | 62 + .../core/main/ServiceLaunchException.java | 73 + .../slider/core/main/ServiceLauncher.java | 642 + .../slider/core/main/ServiceShutdownHook.java | 80 + .../core/persist/AggregateConfSerDeser.java | 55 + .../core/persist/AppDefinitionPersister.java | 260 + .../core/persist/ApplicationReportSerDeser.java | 57 + .../slider/core/persist/ConfPersister.java | 286 + .../slider/core/persist/ConfTreeSerDeser.java | 54 + .../apache/slider/core/persist/Filenames.java | 28 + .../slider/core/persist/InstancePaths.java | 58 + .../slider/core/persist/JsonSerDeser.java | 243 + .../persist/LockAcquireFailedException.java | 40 + .../slider/core/persist/LockHeldAction.java | 38 + .../apache/slider/core/persist/PersistKeys.java | 25 + .../core/registry/SliderRegistryUtils.java | 62 + .../slider/core/registry/YarnAppListClient.java | 189 + .../core/registry/docstore/ConfigFormat.java | 60 + .../core/registry/docstore/ConfigUtils.java | 96 + .../docstore/ConfigurationResolver.java | 24 + .../core/registry/docstore/ExportEntry.java | 120 + .../registry/docstore/PublishedConfigSet.java | 100 + .../docstore/PublishedConfiguration.java | 196 + .../PublishedConfigurationOutputter.java | 210 + .../registry/docstore/PublishedExports.java | 140 + .../docstore/PublishedExportsOutputter.java | 104 + .../registry/docstore/PublishedExportsSet.java | 98 + .../slider/core/registry/docstore/UriMap.java | 38 + .../registry/info/CustomRegistryConstants.java | 57 + .../core/registry/retrieve/AMWebClient.java | 158 + .../registry/retrieve/RegistryRetriever.java | 183 + .../core/restclient/HttpOperationResponse.java | 34 + .../apache/slider/core/restclient/HttpVerb.java | 57 + .../restclient/SliderURLConnectionFactory.java | 176 + .../core/restclient/UgiJerseyBinding.java | 154 + .../restclient/UrlConnectionOperations.java | 210 + .../slider/core/zk/BlockingZKWatcher.java | 67 + .../slider/core/zk/MiniZooKeeperCluster.java | 423 + .../org/apache/slider/core/zk/ZKCallback.java | 31 + .../apache/slider/core/zk/ZKIntegration.java | 323 + .../apache/slider/core/zk/ZKPathBuilder.java | 82 + .../apache/slider/core/zk/ZookeeperUtils.java | 147 + .../providers/AbstractClientProvider.java | 248 + .../providers/AbstractProviderService.java | 424 + .../apache/slider/providers/MonitorDetail.java | 43 + .../slider/providers/PlacementPolicy.java | 64 + .../providers/PlacementPolicyOptions.java | 26 + .../slider/providers/ProviderCompleted.java | 29 + .../providers/ProviderCompletedCallable.java | 38 + .../apache/slider/providers/ProviderCore.java | 43 + .../apache/slider/providers/ProviderRole.java | 135 + .../slider/providers/ProviderService.java | 217 + .../apache/slider/providers/ProviderUtils.java | 530 + .../slider/providers/SliderProviderFactory.java | 110 + .../providers/agent/AgentClientProvider.java | 701 + .../slider/providers/agent/AgentKeys.java | 109 + .../providers/agent/AgentLaunchParameter.java | 130 + .../providers/agent/AgentProviderFactory.java | 47 + .../providers/agent/AgentProviderService.java | 3212 ++ .../slider/providers/agent/AgentRoles.java | 38 + .../slider/providers/agent/AgentUtils.java | 134 + .../apache/slider/providers/agent/Command.java | 59 + .../slider/providers/agent/CommandResult.java | 40 + .../providers/agent/ComponentCommandOrder.java | 181 + .../providers/agent/ComponentInstanceState.java | 340 + .../providers/agent/ComponentTagProvider.java | 127 + .../slider/providers/agent/ContainerState.java | 41 + .../providers/agent/HeartbeatMonitor.java | 130 + .../apache/slider/providers/agent/State.java | 199 + .../application/metadata/AbstractComponent.java | 80 + .../metadata/AbstractMetainfoParser.java | 130 + .../metadata/AbstractMetainfoSchema.java | 69 + .../metadata/AddonPackageMetainfoParser.java | 53 + .../agent/application/metadata/Application.java | 193 + .../metadata/ApplicationPackage.java | 69 + .../application/metadata/CommandOrder.java | 61 + .../application/metadata/CommandScript.java | 72 + .../agent/application/metadata/Component.java | 217 + .../application/metadata/ComponentCommand.java | 85 + .../application/metadata/ComponentExport.java | 54 + .../metadata/ComponentsInAddonPackage.java | 26 + .../agent/application/metadata/ConfigFile.java | 59 + .../application/metadata/DefaultConfig.java | 39 + .../metadata/DefaultConfigParser.java | 54 + .../application/metadata/DockerContainer.java | 187 + .../metadata/DockerContainerInputFile.java | 50 + .../metadata/DockerContainerMount.java | 60 + .../metadata/DockerContainerPort.java | 66 + .../agent/application/metadata/Export.java | 61 + .../agent/application/metadata/ExportGroup.java | 71 + .../agent/application/metadata/Metainfo.java | 118 + .../application/metadata/MetainfoParser.java | 97 + .../agent/application/metadata/OSPackage.java | 51 + .../agent/application/metadata/OSSpecific.java | 57 + .../agent/application/metadata/Package.java | 60 + .../application/metadata/PropertyInfo.java | 54 + .../agent/application/metadata/Validate.java | 27 + .../org/apache/slider/providers/agent/todo.md | 22 + .../slideram/SliderAMClientProvider.java | 304 + .../slideram/SliderAMProviderService.java | 189 + .../appmaster/AppMasterActionOperations.java | 29 + .../server/appmaster/PrivilegedConnectToCM.java | 48 + .../appmaster/ProtobufClusterServices.java | 36 + .../server/appmaster/PublishedArtifacts.java | 31 + .../server/appmaster/RoleLaunchService.java | 257 + .../server/appmaster/SliderAppMaster.java | 2450 ++ .../appmaster/actions/ActionFlexCluster.java | 44 + .../server/appmaster/actions/ActionHalt.java | 53 + .../appmaster/actions/ActionKillContainer.java | 86 + .../actions/ActionRegisterServiceInstance.java | 59 + .../appmaster/actions/ActionStartContainer.java | 62 + .../appmaster/actions/ActionStopQueue.java | 56 + .../appmaster/actions/ActionStopSlider.java | 162 + .../actions/ActionUpgradeContainers.java | 106 + .../server/appmaster/actions/AsyncAction.java | 138 + .../actions/EscalateOutstandingRequests.java | 45 + .../actions/ProviderReportedContainerLoss.java | 53 + .../actions/ProviderStartupCompleted.java | 36 + .../server/appmaster/actions/QueueAccess.java | 72 + .../server/appmaster/actions/QueueExecutor.java | 90 + .../server/appmaster/actions/QueueService.java | 202 + .../actions/RegisterComponentInstance.java | 59 + .../appmaster/actions/RenewingAction.java | 141 + .../appmaster/actions/ResetFailureWindow.java | 39 + .../actions/ReviewAndFlexApplicationSize.java | 43 + .../actions/UnregisterComponentInstance.java | 51 + .../server/appmaster/management/BoolMetric.java | 87 + .../management/BoolMetricPredicate.java | 44 + .../server/appmaster/management/LongGauge.java | 98 + .../management/LongMetricFunction.java | 44 + .../appmaster/management/MeterAndCounter.java | 109 + .../management/MetricsAndMonitoring.java | 195 + .../management/MetricsBindingService.java | 151 + .../appmaster/management/MetricsConstants.java | 58 + .../appmaster/management/MetricsKeys.java | 92 + .../management/PrefixedMetricsSet.java | 53 + .../management/RangeLimitedCounter.java | 85 + .../appmaster/management/RecordedEvent.java | 58 + .../server/appmaster/management/Timestamp.java | 33 + .../management/YarnServiceHealthCheck.java | 38 + .../server/appmaster/monkey/ChaosEntry.java | 85 + .../server/appmaster/monkey/ChaosKillAM.java | 48 + .../appmaster/monkey/ChaosKillContainer.java | 84 + .../appmaster/monkey/ChaosMonkeyService.java | 138 + .../server/appmaster/monkey/ChaosTarget.java | 24 + .../appmaster/monkey/MonkeyPlayAction.java | 48 + .../operations/AbstractRMOperation.java | 30 + .../operations/AsyncRMOperationHandler.java | 110 + .../operations/CancelSingleRequest.java | 54 + .../operations/ContainerReleaseOperation.java | 47 + .../operations/ContainerRequestOperation.java | 62 + .../ProviderNotifyingOperationHandler.java | 55 + .../operations/RMOperationHandler.java | 32 + .../operations/RMOperationHandlerActions.java | 60 + .../slider/server/appmaster/rpc/RpcBinder.java | 310 + .../appmaster/rpc/SliderAMPolicyProvider.java | 41 + .../appmaster/rpc/SliderClusterProtocolPB.java | 27 + .../rpc/SliderClusterProtocolPBImpl.java | 318 + .../rpc/SliderClusterProtocolProxy.java | 358 + .../server/appmaster/rpc/SliderIPCService.java | 551 + .../appmaster/rpc/SliderRPCSecurityInfo.java | 87 + .../security/SecurityConfiguration.java | 162 + .../state/AbstractClusterServices.java | 61 + .../slider/server/appmaster/state/AppState.java | 2489 ++ .../appmaster/state/AppStateBindingInfo.java | 63 + .../state/ContainerAllocationOutcome.java | 44 + .../state/ContainerAllocationResults.java | 50 + .../appmaster/state/ContainerAssignment.java | 60 + .../appmaster/state/ContainerOutcome.java | 61 + .../appmaster/state/ContainerPriority.java | 109 + .../state/ContainerReleaseSelector.java | 37 + .../MostRecentContainerReleaseSelector.java | 51 + .../server/appmaster/state/NodeEntry.java | 325 + .../server/appmaster/state/NodeInstance.java | 409 + .../slider/server/appmaster/state/NodeMap.java | 174 + .../appmaster/state/OutstandingRequest.java | 428 + .../state/OutstandingRequestTracker.java | 482 + .../appmaster/state/ProviderAppState.java | 307 + .../server/appmaster/state/RoleHistory.java | 1101 + .../appmaster/state/RoleHistoryUtils.java | 50 + .../appmaster/state/RoleHostnamePair.java | 75 + .../server/appmaster/state/RoleInstance.java | 323 + .../server/appmaster/state/RoleStatus.java | 563 + .../appmaster/state/SimpleReleaseSelector.java | 33 + .../state/StateAccessForProviders.java | 313 + .../server/appmaster/web/AgentService.java | 37 + .../server/appmaster/web/HttpCacheHeaders.java | 35 + .../appmaster/web/SliderAMController.java | 69 + .../server/appmaster/web/SliderAMWebApp.java | 108 + .../web/SliderDefaultWrapperServlet.java | 48 + .../slider/server/appmaster/web/WebAppApi.java | 95 + .../server/appmaster/web/WebAppApiImpl.java | 128 + .../server/appmaster/web/layout/AppLayout.java | 32 + .../web/layout/ClusterSpecificationView.java | 32 + .../web/layout/ContainerStatsView.java | 33 + .../appmaster/web/layout/WebUILayout.java | 43 + .../appmaster/web/rest/AMWadlGenerator.java | 72 + .../web/rest/AMWadlGeneratorConfig.java | 34 + .../appmaster/web/rest/AMWebServices.java | 72 + .../web/rest/AbstractSliderResource.java | 157 + .../appmaster/web/rest/InsecureAmFilter.java | 105 + .../web/rest/InsecureAmFilterInitializer.java | 102 + .../server/appmaster/web/rest/RestPaths.java | 172 + .../web/rest/SliderJacksonJaxbJsonProvider.java | 58 + .../web/rest/agent/AgentCommandType.java | 23 + .../appmaster/web/rest/agent/AgentEnv.java | 376 + .../appmaster/web/rest/agent/AgentResource.java | 118 + .../web/rest/agent/AgentRestOperations.java | 28 + .../appmaster/web/rest/agent/AgentWebApp.java | 258 + .../web/rest/agent/AgentWebServices.java | 40 + .../appmaster/web/rest/agent/CommandReport.java | 207 + .../web/rest/agent/ComponentStatus.java | 129 + .../appmaster/web/rest/agent/DiskInfo.java | 128 + .../web/rest/agent/ExecutionCommand.java | 310 + .../appmaster/web/rest/agent/HeartBeat.java | 149 + .../web/rest/agent/HeartBeatResponse.java | 147 + .../appmaster/web/rest/agent/HostInfo.java | 398 + .../appmaster/web/rest/agent/HostStatus.java | 63 + .../appmaster/web/rest/agent/Register.java | 193 + .../web/rest/agent/RegistrationCommand.java | 43 + .../web/rest/agent/RegistrationResponse.java | 133 + .../web/rest/agent/RegistrationStatus.java | 22 + .../appmaster/web/rest/agent/StatusCommand.java | 152 + .../ApplicationResouceContentCacheFactory.java | 63 + .../rest/application/ApplicationResource.java | 516 + .../application/actions/RestActionPing.java | 50 + .../application/actions/RestActionStop.java | 67 + .../rest/application/actions/StopResponse.java | 29 + .../web/rest/application/package-info.java | 24 + .../resources/AggregateModelRefresher.java | 47 + .../application/resources/AppconfRefresher.java | 55 + .../application/resources/CachedContent.java | 121 + .../application/resources/ContentCache.java | 67 + .../resources/LiveComponentsRefresher.java | 39 + .../resources/LiveContainersRefresher.java | 52 + .../resources/LiveNodesRefresher.java | 41 + .../resources/LiveResourcesRefresher.java | 68 + .../resources/LiveStatisticsRefresher.java | 39 + .../resources/ResourceRefresher.java | 31 + .../resources/ResourceSnapshotRefresher.java | 40 + .../web/rest/management/ManagementResource.java | 93 + .../management/resources/ActionsResource.java | 22 + .../resources/AggregateConfResource.java | 90 + .../management/resources/ComponentResource.java | 53 + .../management/resources/ConfTreeResource.java | 69 + .../management/resources/ResourceFactory.java | 47 + .../web/rest/publisher/PublisherResource.java | 273 + .../web/rest/registry/PathEntryResource.java | 45 + .../web/rest/registry/RegistryResource.java | 151 + .../web/view/ClusterSpecificationBlock.java | 55 + .../appmaster/web/view/ContainerStatsBlock.java | 282 + .../server/appmaster/web/view/IndexBlock.java | 305 + .../server/appmaster/web/view/NavBlock.java | 62 + .../appmaster/web/view/SliderHamletBlock.java | 56 + .../slider/server/avro/LoadedRoleHistory.java | 92 + .../slider/server/avro/NewerFilesFirst.java | 43 + .../slider/server/avro/OlderFilesFirst.java | 43 + .../slider/server/avro/RoleHistoryWriter.java | 449 + .../slider/server/servicemonitor/HttpProbe.java | 82 + .../server/servicemonitor/LogEntryBuilder.java | 76 + .../server/servicemonitor/MonitorKeys.java | 279 + .../server/servicemonitor/MonitorUtils.java | 109 + .../slider/server/servicemonitor/PortProbe.java | 107 + .../slider/server/servicemonitor/Probe.java | 107 + .../servicemonitor/ProbeFailedException.java | 32 + .../ProbeInterruptedException.java | 29 + .../server/servicemonitor/ProbePhase.java | 56 + .../servicemonitor/ProbeReportHandler.java | 79 + .../server/servicemonitor/ProbeStatus.java | 173 + .../server/servicemonitor/ProbeWorker.java | 446 + .../server/servicemonitor/ReportingLoop.java | 265 + .../servicemonitor/YarnApplicationProbe.java | 90 + .../AbstractSecurityStoreGenerator.java | 98 + .../services/security/CertificateManager.java | 495 + .../services/security/KeystoreGenerator.java | 64 + .../server/services/security/SecurityStore.java | 66 + .../security/SecurityStoreGenerator.java | 40 + .../server/services/security/SecurityUtils.java | 256 + .../services/security/SignCertResponse.java | 67 + .../server/services/security/SignMessage.java | 54 + .../services/security/StoresGenerator.java | 68 + .../services/security/TruststoreGenerator.java | 62 + .../utility/AbstractSliderLaunchedService.java | 120 + .../services/utility/EndOfServiceWaiter.java | 87 + .../LaunchedWorkflowCompositeService.java | 117 + .../services/utility/PatternValidator.java | 61 + .../server/services/utility/WebAppService.java | 69 + .../services/workflow/ClosingService.java | 94 + .../services/workflow/ForkedProcessService.java | 301 + .../services/workflow/LongLivedProcess.java | 598 + .../LongLivedProcessLifecycleEvent.java | 41 + .../server/services/workflow/ServiceParent.java | 44 + .../workflow/ServiceTerminatingCallable.java | 92 + .../workflow/ServiceTerminatingRunnable.java | 72 + .../services/workflow/ServiceThreadFactory.java | 102 + .../workflow/WorkflowCallbackService.java | 113 + .../workflow/WorkflowCompositeService.java | 167 + .../workflow/WorkflowExecutorService.java | 113 + .../services/workflow/WorkflowRpcService.java | 76 + .../WorkflowScheduledExecutorService.java | 38 + .../workflow/WorkflowSequenceService.java | 306 + .../server/services/workflow/package-info.java | 172 + .../YarnRegistryViewForProviders.java | 270 + .../src/main/proto/SliderClusterMessages.proto | 396 + .../src/main/proto/SliderClusterProtocol.proto | 184 + .../org.apache.hadoop.security.SecurityInfo | 15 + .../org/apache/slider/log4j.properties | 52 + .../slider/providers/agent/conf/agent.txt | 19 + .../slider/providers/agent/conf/command.json | 168 + .../providers/agent/conf/command_template.json | 168 + .../apache/slider/providers/agent/role-node.xml | 65 + .../providers/dynamic/application.properties | 25 + .../providers/slideram/instance/appconf.json | 19 + .../providers/slideram/instance/internal.json | 17 + .../providers/slideram/instance/resources.json | 18 + .../main/resources/org/apache/slider/slider.xml | 30 + .../src/scripts/slider_keytabs.sh | 67 + .../src/scripts/yarnservice.py | 383 + .../hadoop-yarn-slider-core/src/site/site.xml | 26 + .../hadoop-yarn-slider/pom.xml | 39 + .../hadoop-yarn-applications/pom.xml | 2 + 461 files changed, 109486 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml new file mode 100644 index 0000000..14130c5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -0,0 +1,397 @@ + + + 4.0.0 + + org.apache.hadoop + hadoop-yarn-applications + 3.0.0-alpha2-SNAPSHOT + + org.apache.hadoop + hadoop-yarn-slider-core + jar + Apache Hadoop YARN Slider Core + + + + + + src/main/resources + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${java.home} + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-sources + + schema + + + ${project.basedir}/src/main/avro/ + + ${project.build.directory}/generated-sources/java + + + + + + + + + + + com.beust + jcommander + 1.30 + + + + org.slf4j + slf4j-api + + + + log4j + log4j + runtime + + + + com.google.guava + guava + + + + org.codehaus.jackson + jackson-core-asl + compile + + + + org.codehaus.jackson + jackson-jaxrs + compile + + + + org.codehaus.jackson + jackson-mapper-asl + compile + + + + org.codehaus.jackson + jackson-xc + compile + + + + org.apache.hadoop + hadoop-hdfs + + + + org.apache.hadoop + hadoop-yarn-client + compile + + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + compile + + + + org.apache.hadoop + hadoop-yarn-registry + compile + + + + junit + junit + test + + + + com.google.protobuf + protobuf-java + + + + org.apache.avro + avro + + + + org.apache.commons + commons-compress + + + + commons-digester + commons-digester + 1.8 + + + + commons-io + commons-io + + + + commons-lang + commons-lang + + + + commons-logging + commons-logging + + + + com.codahale.metrics + metrics-core + + + + com.codahale.metrics + metrics-servlets + 3.0.1 + + + + + + + + org.apache.zookeeper + zookeeper + + + + + + + + javax.servlet + servlet-api + + + + javax.xml.bind + jaxb-api + + + + com.sun.jersey + jersey-client + + + + com.sun.jersey + jersey-json + + + + com.sun.jersey + jersey-server + + + + com.google.inject + guice + + + + com.google.code.gson + gson + + + + com.google.inject.extensions + guice-servlet + + + + com.sun.jersey.contribs + jersey-guice + + + + org.mockito + mockito-all + test + + + + org.mortbay.jetty + jetty + + + + org.mortbay.jetty + jetty-util + + + + org.mortbay.jetty + jetty-sslengine + + + + javax.servlet.jsp + jsp-api + runtime + + + + org.codehaus.jettison + jettison + + + org.mortbay.jetty + jetty-sslengine + 6.1.26 + + + + org.yaml + snakeyaml + 1.16 + compile + + + + + + + + compile-protobuf + + + + org.apache.hadoop + hadoop-maven-plugins + + + compile-protoc + generate-sources + + protoc + + + ${protobuf.version} + protoc + + ${basedir}/src/main/proto + + + ${basedir}/src/main/proto + + SliderClusterMessages.proto + SliderClusterProtocol.proto + + + ${basedir}/src/main/java + + + + + + + + + + + rat + + + + + org.apache.rat + apache-rat-plugin + + + check-licenses + + check + + + + + + **/*.json + src/test/python/agent.ini + src/test/python/version + **/THIRD-PARTY.properties + src/main/resources/webapps/slideram/.keep + src/main/resources/webapps/slideragent/.keep + src/main/resources/webapps/static/yarn.dt.plugins.js + + src/main/resources/webapps/static/dt-1.9.4/** + + src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js + + src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js + src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css + + src/main/resources/webapps/static/jt/jquery.jstree.js + + src/main/java/org/apache/slider/api/proto/Messages.java + src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java + src/test/app_packages/test_am_config/resources/test.template + src/test/app_packages/test_am_config/test_archive/testfile + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml new file mode 100644 index 0000000..23383c8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/assembly/executable-jar.xml @@ -0,0 +1,47 @@ + + + + distribution + + zip + + true + + + ${project.build.directory} + / + + *.jar + + + + + + runtime + /lib + + false + + false + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties new file mode 100644 index 0000000..1abd56e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties @@ -0,0 +1,33 @@ +# Generated by org.codehaus.mojo.license.AddThirdPartyMojo +#------------------------------------------------------------------------------- +# Already used licenses in project : +# - Apache License +# - BSD +# - CDDL + GPLv2 with classpath exception +# - CDDL 1.1 +# - CDDL License +# - CDDL+GPL +# - Common Public License Version 1.0 +# - Eclipse Public License - Version 1.0 +# - GNU Lesser General Public License (LGPL), Version 2.1 +# - GNU Lesser General Public License, Version 2.1 +# - GPL2 w/ CPE +# - MIT License +# - MPL 1.1 +# - New BSD License +# - Public Domain +# - Revised BSD +# - The Apache Software License, Version 2.0 +# - The BSD 3-Clause License +# - The BSD License +# - The MIT License +#------------------------------------------------------------------------------- +# Please fill the missing licenses for dependencies : +# +# +#Thu Oct 15 16:45:02 EDT 2015 +commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0 +javax.servlet--servlet-api--2.5=CDDL License +javax.servlet.jsp--jsp-api--2.1=CDDL License +org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0 +org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0 http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc new file mode 100644 index 0000000..3667c01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/avro/org/apache/slider/server/avro/RoleHistoryRecord.avsc @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +[ + + { + "type": "record", + "name": "NodeEntryRecord", + "namespace": "org.apache.slider.server.avro", + "fields": [ + { + "name": "host", + "type": "string" + }, + { + "name": "role", + "type": "int" + }, + { + "name": "active", + "type": "boolean" + }, + { + "name": "last_used", + "type": "long" + } + ] + }, + + { + "type": "record", + "name": "RoleHistoryHeader", + "namespace": "org.apache.slider.server.avro", + "fields": [ + { + "name": "version", + "type": "int" + }, + { + "name": "saved", + "type": "long" + }, + { + "name": "savedx", + "type": "string" + }, + { + "name": "savedate", + "type": "string", + "default": "" + }, + { + "name": "roles", + "type": "int" + } + ] + }, + { + "type": "record", + "name": "RoleHistoryMapping", + "namespace": "org.apache.slider.server.avro", + "fields": [ + { + "name": "rolemap", + "type": { + "type": "map", + "values": "int" + } + } + ] + }, + { + "type": "record", + "name": "RoleHistoryFooter", + "namespace": "org.apache.slider.server.avro", + "fields": [ + { + "name": "count", + "type": "long" + } + ] + }, + + { + "type": "record", + "name": "RoleHistoryRecord", + "namespace": "org.apache.slider.server.avro", + "fields": [ + { + "name": "entry", + "type": [ + "org.apache.slider.server.avro.NodeEntryRecord", + "org.apache.slider.server.avro.RoleHistoryHeader", + "org.apache.slider.server.avro.RoleHistoryFooter", + "org.apache.slider.server.avro.RoleHistoryMapping" + ] + } + ] + } + +] http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java new file mode 100644 index 0000000..8c572b3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.InvocationTargetException; +import java.net.InetAddress; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.regex.Pattern; + +import static org.apache.hadoop.security.UserGroupInformation.*; +import static org.apache.hadoop.security.authentication.util.KerberosUtil.*; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; + +/** + * Kerberos diagnostics + * At some point this may move to hadoop core, so please keep use of slider + * methods and classes to ~0. + * + * This operation expands some of the diagnostic output of the security code, + * but not all. For completeness + * + * Set the environment variable {@code HADOOP_JAAS_DEBUG=true} + * Set the log level for {@code org.apache.hadoop.security=DEBUG} + */ +public class KerberosDiags implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(KerberosDiags.class); + public static final String KRB5_CCNAME = "KRB5CCNAME"; + public static final String JAVA_SECURITY_KRB5_CONF + = "java.security.krb5.conf"; + public static final String JAVA_SECURITY_KRB5_REALM + = "java.security.krb5.realm"; + public static final String SUN_SECURITY_KRB5_DEBUG + = "sun.security.krb5.debug"; + public static final String SUN_SECURITY_SPNEGO_DEBUG + = "sun.security.spnego.debug"; + public static final String SUN_SECURITY_JAAS_FILE + = "java.security.auth.login.config"; + public static final String KERBEROS_KINIT_COMMAND + = "hadoop.kerberos.kinit.command"; + public static final String HADOOP_AUTHENTICATION_IS_DISABLED + = "Hadoop authentication is disabled"; + public static final String UNSET = "(unset)"; + public static final String NO_DEFAULT_REALM = "Cannot locate default realm"; + + private final Configuration conf; + private final List services; + private final PrintWriter out; + private final File keytab; + private final String principal; + private final long minKeyLength; + private final boolean securityRequired; + + public static final String CAT_JVM = "JVM"; + public static final String CAT_JAAS = "JAAS"; + public static final String CAT_CONFIG = "CONFIG"; + public static final String CAT_LOGIN = "LOGIN"; + public static final String CAT_KERBEROS = "KERBEROS"; + public static final String CAT_SASL = "SASL"; + + @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") + public KerberosDiags(Configuration conf, + PrintWriter out, + List services, + File keytab, + String principal, + long minKeyLength, + boolean securityRequired) { + this.conf = conf; + this.services = services; + this.keytab = keytab; + this.principal = principal; + this.out = out; + this.minKeyLength = minKeyLength; + this.securityRequired = securityRequired; + } + + @Override + public void close() throws IOException { + flush(); + } + + /** + * Execute diagnostics. + *

+ * Things it would be nice if UGI made accessible + *

    + *
  1. A way to enable JAAS debug programatically
    2. + *
  2. Access to the TGT
    3. + *
+ * @return true if security was enabled and all probes were successful + * @throws KerberosDiagsFailure explicitly raised failure + * @throws Exception other security problems + */ + @SuppressWarnings("deprecation") + public boolean execute() throws Exception { + + title("Kerberos Diagnostics scan at %s", + new Date(System.currentTimeMillis())); + + // check that the machine has a name + println("Hostname: %s", + InetAddress.getLocalHost().getCanonicalHostName()); + + // Fail fast on a JVM without JCE installed. + validateKeyLength(); + + // look at realm + println("JVM Kerberos Login Module = %s", getKrb5LoginModuleName()); + printDefaultRealm(); + + title("System Properties"); + for (String prop : new String[]{ + JAVA_SECURITY_KRB5_CONF, + JAVA_SECURITY_KRB5_REALM, + SUN_SECURITY_KRB5_DEBUG, + SUN_SECURITY_SPNEGO_DEBUG, + SUN_SECURITY_JAAS_FILE + }) { + printSysprop(prop); + } + + title("Environment Variables"); + for (String env : new String[]{ + "HADOOP_JAAS_DEBUG", + KRB5_CCNAME, + HADOOP_USER_NAME, + HADOOP_PROXY_USER, + HADOOP_TOKEN_FILE_LOCATION, + }) { + printEnv(env); + } + + for (String prop : new String[]{ + KERBEROS_KINIT_COMMAND, + HADOOP_SECURITY_AUTHENTICATION, + HADOOP_SECURITY_AUTHORIZATION, + "hadoop.kerberos.min.seconds.before.relogin", // not in 2.6 + "hadoop.security.dns.interface", // not in 2.6 + "hadoop.security.dns.nameserver", // not in 2.6 + HADOOP_RPC_PROTECTION, + HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS, + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX, + HADOOP_SECURITY_GROUP_MAPPING, + "hadoop.security.impersonation.provider.class", // not in 2.6 + "dfs.data.transfer.protection" // HDFS + }) { + printConfOpt(prop); + } + + // check that authentication is enabled + if (SecurityUtil.getAuthenticationMethod(conf) + .equals(AuthenticationMethod.SIMPLE)) { + println(HADOOP_AUTHENTICATION_IS_DISABLED); + failif(securityRequired, CAT_CONFIG, HADOOP_AUTHENTICATION_IS_DISABLED); + // no security, skip rest of test + return false; + } + + validateKrb5File(); + validateSasl(HADOOP_SECURITY_SASL_PROPS_RESOLVER_CLASS); + validateSasl("dfs.data.transfer.saslproperties.resolver.class"); + validateKinitExecutable(); + validateJAAS(); + // now the big test: login, then try again + boolean krb5Debug = getAndSet(SUN_SECURITY_KRB5_DEBUG); + boolean spnegoDebug = getAndSet(SUN_SECURITY_SPNEGO_DEBUG); + try { + title("Logging in"); + + if (keytab != null) { + dumpKeytab(keytab); + loginFromKeytab(); + } else { + UserGroupInformation loginUser = getLoginUser(); + dumpUGI("Log in user", loginUser); + validateUGI("Login user", loginUser); + println("Ticket based login: %b", isLoginTicketBased()); + println("Keytab based login: %b", isLoginKeytabBased()); + } + + return true; + } finally { + // restore original system properties + System.setProperty(SUN_SECURITY_KRB5_DEBUG, + Boolean.toString(krb5Debug)); + System.setProperty(SUN_SECURITY_SPNEGO_DEBUG, + Boolean.toString(spnegoDebug)); + } + } + + /** + * Fail fast on a JVM without JCE installed. + * + * This is a recurrent problem + * (that is: it keeps creeping back with JVM updates); + * a fast failure is the best tactic + * @throws NoSuchAlgorithmException + */ + + protected void validateKeyLength() throws NoSuchAlgorithmException { + int aesLen = Cipher.getMaxAllowedKeyLength("AES"); + println("Maximum AES encryption key length %d bits", aesLen); + failif (aesLen < minKeyLength, + CAT_JVM, + "Java Cryptography Extensions are not installed on this JVM." + +" Maximum supported key length %s - minimum required %d", + aesLen, minKeyLength); + } + + /** + * Get the default realm. + *

+ * Not having a default realm may be harmless, so is noted at info. + * All other invocation failures are downgraded to warn, as + * follow-on actions may still work. + * failure to invoke the method via introspection is rejected, + * as it's a sign of JVM compatibility issues that may have other + * consequences + */ + protected void printDefaultRealm() { + try { + println("Default Realm = %s", + getDefaultRealm()); + } catch (ClassNotFoundException + | IllegalAccessException + | NoSuchMethodException e) { + + throw new KerberosDiagsFailure(CAT_JVM, e, + "Failed to invoke krb5.Config.getDefaultRealm: %s", e); + } catch (InvocationTargetException e) { + Throwable cause = e.getCause() != null ? e.getCause() : e; + if (cause.toString().contains(NO_DEFAULT_REALM)) { + // exception raised if there is no default realm. This is not + // always a problem, so downgrade to a message. + println("Host has no default realm"); + LOG.debug(cause.toString(), cause); + } else { + println("Kerberos.getDefaultRealm() failed: %s\n%s", + cause, + org.apache.hadoop.util.StringUtils.stringifyException(cause)); + } + } + } + + /** + * Locate the krb5.conf file and dump it. + * No-op on windows. + * @throws IOException + */ + private void validateKrb5File() throws IOException { + if (!Shell.WINDOWS) { + title("Locating Kerberos configuration file"); + String krbPath = "/etc/krb5.conf"; + String jvmKrbPath = System.getProperty(JAVA_SECURITY_KRB5_CONF); + if (jvmKrbPath != null) { + println("Setting kerberos path from sysprop %s: %s", + JAVA_SECURITY_KRB5_CONF, jvmKrbPath); + krbPath = jvmKrbPath; + } + + String krb5name = System.getenv(KRB5_CCNAME); + if (krb5name != null) { + println("Setting kerberos path from environment variable %s: %s", + KRB5_CCNAME, krb5name); + krbPath = krb5name; + if (jvmKrbPath != null) { + println("Warning - both %s and %s were set - %s takes priority", + JAVA_SECURITY_KRB5_CONF, KRB5_CCNAME, KRB5_CCNAME); + } + } + + File krbFile = new File(krbPath); + println("Kerberos configuration file = %s", krbFile); + failif(!krbFile.exists(), + CAT_KERBEROS, + "Kerberos configuration file %s not found", krbFile); + dump(krbFile); + } + } + + /** + * Dump a keytab: list all principals. + * @param keytabFile the keytab file + * @throws IOException IO problems + */ + public void dumpKeytab(File keytabFile) throws IOException { + title("Examining keytab %s", keytabFile); + File kt = keytabFile.getCanonicalFile(); + failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt); + failif(!kt.isFile(), CAT_CONFIG, "Keytab is not a valid file: %s", kt); + + String[] names = getPrincipalNames(keytabFile.getCanonicalPath(), + Pattern.compile(".*")); + println("keytab entry count: %d", names.length); + for (String name : names) { + println(" %s", name); + } + println("-----"); + } + + /** + * Log in from a keytab, dump the UGI, validate it, then try and log in again. + * That second-time login catches JVM/Hadoop compatibility problems. + * @throws IOException + */ + private void loginFromKeytab() throws IOException { + UserGroupInformation ugi; + String identity; + if (keytab != null) { + File kt = keytab.getCanonicalFile(); + println("Using keytab %s principal %s", kt, principal); + identity = principal; + + failif(StringUtils.isEmpty(principal), CAT_KERBEROS, + "No principal defined"); + ugi = loginUserFromKeytabAndReturnUGI(principal, kt.getPath()); + dumpUGI(identity, ugi); + validateUGI(principal, ugi); + + title("Attempting to log in from keytab again"); + // package scoped -hence the reason why this class must be in the + // hadoop.security package + setShouldRenewImmediatelyForTests(true); + // attempt a new login + ugi.reloginFromKeytab(); + } else { + println("No keytab: logging is as current user"); + } + } + + /** + * Dump a UGI. + * @param title title of this section + * @param ugi UGI to dump + * @throws IOException + */ + private void dumpUGI(String title, UserGroupInformation ugi) + throws IOException { + title(title); + println("UGI instance = %s", ugi); + println("Has kerberos credentials: %b", ugi.hasKerberosCredentials()); + println("Authentication method: %s", ugi.getAuthenticationMethod()); + println("Real Authentication method: %s", + ugi.getRealAuthenticationMethod()); + title("Group names"); + for (String name : ugi.getGroupNames()) { + println(name); + } + title("Credentials"); + Credentials credentials = ugi.getCredentials(); + List secretKeys = credentials.getAllSecretKeys(); + title("Secret keys"); + if (!secretKeys.isEmpty()) { + for (Text secret: secretKeys) { + println("%s", secret); + } + } else { + println("(none)"); + } + + dumpTokens(ugi); + } + + /** + * Validate the UGI: verify it is kerberized. + * @param messagePrefix message in exceptions + * @param user user to validate + */ + private void validateUGI(String messagePrefix, UserGroupInformation user) { + failif(!user.hasKerberosCredentials(), + CAT_LOGIN, "%s: No kerberos credentials for %s", messagePrefix, user); + failif(user.getAuthenticationMethod() == null, + CAT_LOGIN, "%s: Null AuthenticationMethod for %s", messagePrefix, user); + } + + /** + * A cursory look at the {@code kinit} executable. + * If it is an absolute path: it must exist with a size > 0. + * If it is just a command, it has to be on the path. There's no check + * for that -but the PATH is printed out. + */ + private void validateKinitExecutable() { + String kinit = conf.getTrimmed(KERBEROS_KINIT_COMMAND, ""); + if (!kinit.isEmpty()) { + File kinitPath = new File(kinit); + println("%s = %s", KERBEROS_KINIT_COMMAND, kinitPath); + if (kinitPath.isAbsolute()) { + failif(!kinitPath.exists(), CAT_KERBEROS, + "%s executable does not exist: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + failif(!kinitPath.isFile(), CAT_KERBEROS, + "%s path does not refer to a file: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + failif(kinitPath.length() == 0, CAT_KERBEROS, + "%s file is empty: %s", + KERBEROS_KINIT_COMMAND, kinitPath); + } else { + println("Executable %s is relative -must be on the PATH", kinit); + printEnv("PATH"); + } + } + } + + /** + * Try to load the SASL resolver. + * @param saslPropsResolverKey key for the SASL resolver + */ + private void validateSasl(String saslPropsResolverKey) { + title("Resolving SASL property %s", saslPropsResolverKey); + String saslPropsResolver = conf.getTrimmed(saslPropsResolverKey); + try { + Class resolverClass = conf.getClass( + saslPropsResolverKey, + SaslPropertiesResolver.class, SaslPropertiesResolver.class); + println("Resolver is %s", resolverClass); + } catch (RuntimeException e) { + throw new KerberosDiagsFailure(CAT_SASL, e, + "Failed to load %s class %s", + saslPropsResolverKey, saslPropsResolver); + } + } + + /** + * Validate any JAAS entry referenced in the {@link #SUN_SECURITY_JAAS_FILE} + * property. + */ + private void validateJAAS() { + String jaasFilename = System.getProperty(SUN_SECURITY_JAAS_FILE); + if (jaasFilename != null) { + title("JAAS"); + File jaasFile = new File(jaasFilename); + println("JAAS file is defined in %s: %s", + SUN_SECURITY_JAAS_FILE, jaasFile); + failif(!jaasFile.exists(), CAT_JAAS, + "JAAS file does not exist: %s", jaasFile); + failif(!jaasFile.isFile(), CAT_JAAS, + "Specified JAAS file is not a file: %s", jaasFile); + } + } + + /** + * Dump all tokens of a user + * @param user user + */ + public void dumpTokens(UserGroupInformation user) { + Collection> tokens + = user.getCredentials().getAllTokens(); + title("Token Count: %d", tokens.size()); + for (Token token : tokens) { + println("Token %s", token.getKind()); + } + } + + /** + * Set the System property to true; return the old value for caching + * @param sysprop property + * @return the previous value + */ + private boolean getAndSet(String sysprop) { + boolean old = Boolean.getBoolean(sysprop); + System.setProperty(sysprop, "true"); + return old; + } + + /** + * Flush all active output channels, including {@Code System.err}, + * so as to stay in sync with any JRE log messages. + */ + private void flush() { + if (out != null) { + out.flush(); + } else { + System.out.flush(); + } + System.err.flush(); + } + + /** + * Format and print a line of output. + * This goes to any output file, or + * is logged at info. The output is flushed before and after, to + * try and stay in sync with JRE logging. + * @param format format string + * @param args any arguments + */ + @VisibleForTesting + public void println(String format, Object... args) { + println(format(format, args)); + } + + /** + * Print a line of output. This goes to any output file, or + * is logged at info. The output is flushed before and after, to + * try and stay in sync with JRE logging. + * @param msg message string + */ + @VisibleForTesting + private void println(String msg) { + flush(); + if (out != null) { + out.println(msg); + } else { + LOG.info(msg); + } + flush(); + } + + /** + * Print a title entry + * @param format format string + * @param args any arguments + */ + private void title(String format, Object... args) { + println(""); + println(""); + String msg = "== " + format(format, args) + " =="; + println(msg); + println(""); + } + + /** + * Print a system property, or {@link #UNSET} if unset. + * @param property property to print + */ + private void printSysprop(String property) { + println("%s = \"%s\"", property, + System.getProperty(property, UNSET)); + } + + /** + * Print a configuration option, or {@link #UNSET} if unset. + * @param option option to print + */ + private void printConfOpt(String option) { + println("%s = \"%s\"", option, conf.get(option, UNSET)); + } + + /** + * Print an environment variable's name and value; printing + * {@link #UNSET} if it is not set + * @param variable environment variable + */ + private void printEnv(String variable) { + String env = System.getenv(variable); + println("%s = \"%s\"", variable, env != null ? env : UNSET); + } + + /** + * Dump any file to standard out; add a trailing newline + * @param file file to dump + * @throws IOException IO problems + */ + public void dump(File file) throws IOException { + try (FileInputStream in = new FileInputStream(file)) { + for (String line : IOUtils.readLines(in)) { + println("%s", line); + } + } + println(""); + } + + /** + * Format and raise a failure + * + * @param category category for exception + * @param message string formatting message + * @param args any arguments for the formatting + * @throws KerberosDiagsFailure containing the formatted text + */ + private void fail(String category, String message, Object... args) + throws KerberosDiagsFailure { + throw new KerberosDiagsFailure(category, message, args); + } + + /** + * Conditional failure with string formatted arguments + * @param condition failure condition + * @param category category for exception + * @param message string formatting message + * @param args any arguments for the formatting + * @throws KerberosDiagsFailure containing the formatted text + * if the condition was met + */ + private void failif(boolean condition, + String category, + String message, + Object... args) + throws KerberosDiagsFailure { + if (condition) { + fail(category, message, args); + } + } + + /** + * Format a string, treating a call where there are no varags values + * as a string to pass through unformatted. + * @param message message, which is either a format string + args, or + * a general string + * @param args argument array + * @return a string for printing. + */ + public static String format(String message, Object... args) { + if (args.length == 0) { + return message; + } else { + return String.format(message, args); + } + } + + /** + * Diagnostics failures return the exit code 41, "unauthorized". + * + * They have a category, initially for testing: the category can be + * validated without having to match on the entire string. + */ + public static class KerberosDiagsFailure extends ExitUtil.ExitException { + private final String category; + + public KerberosDiagsFailure(String category, String message) { + super(41, category + ": " + message); + this.category = category; + } + + public KerberosDiagsFailure(String category, String message, Object... args) { + this(category, format(message, args)); + } + + public KerberosDiagsFailure(String category, Throwable throwable, + String message, Object... args) { + this(category, message, args); + initCause(throwable); + } + + public String getCategory() { + return category; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java new file mode 100644 index 0000000..5fc8618 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider; + +import org.apache.slider.client.SliderClient; +import org.apache.slider.core.main.ServiceLauncher; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * This is just the entry point class + */ +public class Slider extends SliderClient { + + + public static final String SERVICE_CLASSNAME = "org.apache.slider.Slider"; + + /** + * This is the main entry point for the service launcher. + * @param args command line arguments. + */ + public static void main(String[] args) { + + //turn the args to a list + List argsList = Arrays.asList(args); + //create a new list, as the ArrayList type doesn't push() on an insert + List extendedArgs = new ArrayList(argsList); + //insert the service name + extendedArgs.add(0, SERVICE_CLASSNAME); + //now have the service launcher do its work + ServiceLauncher.serviceMain(extendedArgs); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org