hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1452668 - in /hadoop/common/branches/branch-1-win: ./ src/packages/win/HadoopServiceHost/ src/packages/win/resources/ src/packages/win/scripts/ src/packages/win/template/conf/
Date Tue, 05 Mar 2013 04:54:55 GMT
Author: suresh
Date: Tue Mar  5 04:54:55 2013
New Revision: 1452668

URL: http://svn.apache.org/r1452668
Log:
HADOOP-9250. Windows installer bugfixes. Contributed by Ivan Mitic.

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/packages/win/HadoopServiceHost/ServiceHost.cs
    hadoop/common/branches/branch-1-win/src/packages/win/resources/winpkg.utils.psm1
    hadoop/common/branches/branch-1-win/src/packages/win/scripts/InstallApi.psm1
    hadoop/common/branches/branch-1-win/src/packages/win/scripts/TestInstallApi.ps1
    hadoop/common/branches/branch-1-win/src/packages/win/scripts/install.ps1
    hadoop/common/branches/branch-1-win/src/packages/win/template/conf/log4j.properties

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Tue Mar  5 04:54:55 2013
@@ -250,6 +250,8 @@ Branch-hadoop-1-win (branched from branc
     HADOOP-9185. TestFileCreation.testFsClose should clean up on exit.
     (Arpit Agarwal via suresh)
 
+    HADOOP-9250. Windows installer bugfixes. (Ivan Mitic via suresh)
+
   Merged from branch-1
 
     HDFS-385. Backport: Add support for an experimental API that allows a

Modified: hadoop/common/branches/branch-1-win/src/packages/win/HadoopServiceHost/ServiceHost.cs
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/HadoopServiceHost/ServiceHost.cs?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/HadoopServiceHost/ServiceHost.cs
(original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/HadoopServiceHost/ServiceHost.cs
Tue Mar  5 04:54:55 2013
@@ -101,6 +101,9 @@ namespace HadoopServiceHost
 
       process.StartInfo.FileName = this.serviceExe;
       process.StartInfo.Arguments = this.serviceArgs;
+      process.StartInfo.WorkingDirectory = AppDomain.CurrentDomain.BaseDirectory;
+      process.StartInfo.CreateNoWindow = false;
+      process.StartInfo.UseShellExecute = false;
 
       if (!process.Start())
       {

Modified: hadoop/common/branches/branch-1-win/src/packages/win/resources/winpkg.utils.psm1
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/resources/winpkg.utils.psm1?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/resources/winpkg.utils.psm1 (original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/resources/winpkg.utils.psm1 Tue Mar
 5 04:54:55 2013
@@ -193,6 +193,35 @@ function Set-ServiceAcl ($service)
     Invoke-Cmd $cmd
 }
 
+# Convenience method for processing command-line credential objects
+# Assumes $credentialsHash is a hash with one of the following being true:
+#  - keys "username" and "password" are set to strings
+#  - key "credentialFilePath" is set to the path of a serialized PSCredential object
+function Get-HadoopUserCredentials($credentialsHash)
+{
+	if($credentialsHash["username"])
+	{
+		Write-Log "Using provided credentials for username $($credentialsHash["username"])" | Out-Null
+		$username = $credentialsHash["username"]
+		if($username -notlike "*\*")
+		{
+			$username = "$ENV:COMPUTERNAME\$username"
+		}
+		$securePassword = $credentialsHash["password"] | ConvertTo-SecureString -AsPlainText -Force
+	}
+	else
+	{
+		Write-Log "Reading credentials from $($credentialsHash['credentialFilePath'])" | Out-Null
+		$import = Import-Clixml -Path $credentialsHash["credentialFilePath"]
+		$username = $import.Username
+		$securePassword = $import.Password | ConvertTo-SecureString
+	}
+	
+	$creds = New-Object System.Management.Automation.PSCredential $username, $securePassword
+	return $creds
+}
+
+Export-ModuleMember -Function Get-HadoopUserCredentials
 Export-ModuleMember -Function Initialize-InstallationEnv
 Export-ModuleMember -Function Invoke-Cmd
 Export-ModuleMember -Function Invoke-CmdChk

Modified: hadoop/common/branches/branch-1-win/src/packages/win/scripts/InstallApi.psm1
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/scripts/InstallApi.psm1?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/scripts/InstallApi.psm1 (original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/scripts/InstallApi.psm1 Tue Mar 
5 04:54:55 2013
@@ -278,16 +278,19 @@ function CreateAndConfigureHadoopService
     $service,
     [String]
     [Parameter( Position=1, Mandatory=$true )]
+    $hdpResourcesDir,
+    [String]
+    [Parameter( Position=2, Mandatory=$true )]
     $serviceBinDir,
     [System.Management.Automation.PSCredential]
-    [Parameter( Position=2, Mandatory=$true )]
+    [Parameter( Position=3, Mandatory=$true )]
     $serviceCredential
 )
 {
     if ( -not ( Get-Service "$service" -ErrorAction SilentlyContinue ) )
     {
         Write-Log "Creating service `"$service`" as $serviceBinDir\$service.exe"
-        $xcopyServiceHost_cmd = "copy /Y `"$ENV:HADOOP_HOME\bin\HadoopServiceHost.exe`" `"$serviceBinDir\$service.exe`""
+        $xcopyServiceHost_cmd = "copy /Y `"$hdpResourcesDir\serviceHost.exe`" `"$serviceBinDir\$service.exe`""
         Invoke-CmdChk $xcopyServiceHost_cmd
 
         #HadoopServiceHost.exe will write to this log but does not create it
@@ -307,7 +310,7 @@ function CreateAndConfigureHadoopService
         $cmd="$ENV:WINDIR\system32\sc.exe failure $service reset= 30 actions= restart/5000"
         Invoke-CmdChk $cmd
 
-        $cmd="$ENV:WINDIR\system32\sc.exe config $service start= auto"
+        $cmd="$ENV:WINDIR\system32\sc.exe config $service start= demand"
         Invoke-CmdChk $cmd
 
         Set-ServiceAcl $service
@@ -406,7 +409,7 @@ function InstallHdfs(
 
     foreach( $service in empty-null $allServices.Split(' '))
     {
-        CreateAndConfigureHadoopService $service $hadoopInstallToBin $serviceCredential
+        CreateAndConfigureHadoopService $service $HDP_RESOURCES_DIR $hadoopInstallToBin $serviceCredential
     }
 
     ###
@@ -545,7 +548,7 @@ function InstallMapRed(
 
     foreach( $service in empty-null $allServices.Split(' '))
     {
-        CreateAndConfigureHadoopService $service $hadoopInstallToBin $serviceCredential
+        CreateAndConfigureHadoopService $service $HDP_RESOURCES_DIR $hadoopInstallToBin $serviceCredential
     }
 
     ###
@@ -920,6 +923,38 @@ function ConfigureHdfs(
     }
 }
 
+### Helper method that extracts all capacity-scheduler configs from the "config"
+### hashmap and applies them to capacity-scheduler.xml.
+### The function returns the list of configs that are not specific to capacity
+### scheduler.
+function ConfigureCapacityScheduler(
+    [string]
+    [parameter( Position=0, Mandatory=$true )]
+    $fileName, 
+    [hashtable]
+    [parameter( Position=1 )]
+    $config = @{} )
+{
+    [hashtable]$newConfig = @{}
+    [hashtable]$newCapSchedulerConfig = @{}
+    foreach( $key in empty-null $config.Keys )
+    {
+        [string]$keyString = $key
+        $value = $config[$key]
+        if ( $keyString.StartsWith("mapred.capacity-scheduler", "CurrentCultureIgnoreCase")
)
+        {
+            $newCapSchedulerConfig.Add($key, $value) > $null
+        }
+        else
+        {
+            $newConfig.Add($key, $value) > $null
+        }
+    }
+
+    UpdateXmlConfig $fileName $newCapSchedulerConfig > $null
+    $newConfig
+}
+
 ###############################################################################
 ###
 ### Alters the configuration of the Hadoop MapRed component.
@@ -951,7 +986,15 @@ function ConfigureMapRed(
     {
         throw "ConfigureMapRed: InstallCore and InstallMapRed must be called before ConfigureMapRed"
     }
-    
+
+    ###
+    ### Apply capacity-scheduler.xml configuration changes. All such configuration properties
+    ### have "mapred.capacity-scheduler" prefix, so it is easy to properly separate them
out
+    ### from other Hadoop configs.
+    ###
+    $capacitySchedulerXmlFile = Join-Path $hadoopInstallToDir "conf\capacity-scheduler.xml"
+    [hashtable]$configs = ConfigureCapacityScheduler $capacitySchedulerXmlFile $configs
+
     ###
     ### Apply configuration changes to mapred-site.xml
     ###

Modified: hadoop/common/branches/branch-1-win/src/packages/win/scripts/TestInstallApi.ps1
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/scripts/TestInstallApi.ps1?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/scripts/TestInstallApi.ps1 (original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/scripts/TestInstallApi.ps1 Tue Mar
 5 04:54:55 2013
@@ -490,6 +490,40 @@ function CoreConfigureWithAliasesTest()
     Uninstall "Core" $NodeInstallRoot
 }
 
+function CoreConfigureCapacitySchedulerTest()
+{
+    Install "core" $NodeInstallRoot $ServiceCredential ""
+
+    $mapRedSiteXml = Join-Path $NodeInstallRoot "hadoop-$HadoopCoreVersion\conf\mapred-site.xml"
+    $capSchedulerSiteXml = Join-Path $NodeInstallRoot "hadoop-$HadoopCoreVersion\conf\capacity-scheduler.xml"
+
+    Install "mapreduce" $NodeInstallRoot $ServiceCredential "jobtracker tasktracker historyserver"
+    Configure "mapreduce" $NodeInstallRoot $ServiceCredential @{
+        "mapred.job.tracker" = "host:port";
+        "mapred.local.dir" = "$NodeInstallRoot\hdfs\mapred\local2";
+        "mapred.capacity-scheduler.prop" = "testcapacity";
+        "mapred.capacity-scheduler.init-worker-threads" = "10"}
+
+    ### Verify that the update took place
+    ValidateXmlConfigValue $mapRedSiteXml "mapred.job.tracker" "host:port"
+    ValidateXmlConfigValue $mapRedSiteXml "mapred.local.dir" "$NodeInstallRoot\hdfs\mapred\local2"
+    ValidateXmlConfigValue $capSchedulerSiteXml "mapred.capacity-scheduler.prop" "testcapacity"
+    ValidateXmlConfigValue $capSchedulerSiteXml "mapred.capacity-scheduler.init-worker-threads"
"10"
+
+    ### Scenario where we only have capacity-scheduler configs
+    Configure "mapreduce" $NodeInstallRoot $ServiceCredential @{
+        "mapred.capacity-scheduler.prop" = "testcapacity2";
+        "mapred.capacity-scheduler.init-worker-threads" = "20"}
+
+    ValidateXmlConfigValue $mapRedSiteXml "mapred.job.tracker" "host:port"
+    ValidateXmlConfigValue $mapRedSiteXml "mapred.local.dir" "$NodeInstallRoot\hdfs\mapred\local2"
+    ValidateXmlConfigValue $capSchedulerSiteXml "mapred.capacity-scheduler.prop" "testcapacity2"
+    ValidateXmlConfigValue $capSchedulerSiteXml "mapred.capacity-scheduler.init-worker-threads"
"20"
+
+    # Cleanup
+    Uninstall "mapreduce" $NodeInstallRoot
+    Uninstall "core" $NodeInstallRoot
+}
 
 try
 {
@@ -521,6 +555,7 @@ try
     TestStartStopServiceRoleNoSupported
     TestProcessAliasConfigOptions
     CoreConfigureWithAliasesTest
+    CoreConfigureCapacitySchedulerTest
 
     # Start/StopService should be tested E2E as it requires all Hadoop binaries
     # to be installed (this test only installs a small subset so that it runs

Modified: hadoop/common/branches/branch-1-win/src/packages/win/scripts/install.ps1
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/scripts/install.ps1?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/scripts/install.ps1 (original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/scripts/install.ps1 Tue Mar  5 04:54:55
2013
@@ -45,11 +45,13 @@
 
 param(
     [String]
-    [Parameter( Position=0, Mandatory=$true )]
+    [Parameter( ParameterSetName='Username', Position=0, Mandatory=$true )]
     $username,
     [String]
-    [Parameter( Position=1, Mandatory=$true )]
+    [Parameter( ParameterSetName='Username', Position=1, Mandatory=$true )]
     $password,
+    [Parameter( ParameterSetName='CredentialFilePath', Mandatory=$true )]
+    $credentialFilePath,
     [String]
     $hdfsRoles = "namenode datanode secondarynamenode",
     [String]
@@ -76,8 +78,14 @@ function Main( $scriptDir )
     Write-Log "nodeInstallRoot: $nodeInstallRoot"
     Write-Log "hadoopInstallToBin: $hadoopInstallToBin"
 
+    ###
+    ### Create the Credential object from the given username and password or the provided
credentials file
+    ###
+    $serviceCredential = Get-HadoopUserCredentials -credentialsHash @{"username" = $username;
"password" = $password; "credentialFilePath" = $credentialFilePath}
+    $username = $serviceCredential.UserName
     Write-Log "Username: $username"
-
+    Write-Log "CredentialFilePath: $credentialFilePath"
+    
     ###
     ### Initialize root directory used for Core, HDFS and MapRed local folders
     ###
@@ -85,12 +93,6 @@ function Main( $scriptDir )
     {
         $ENV:HDFS_DATA_DIR = Join-Path "$ENV:HADOOP_NODE_INSTALL_ROOT" "HDFS"
     }
-    
-    ###
-    ### Create the Credential object from the given username and password
-    ###
-    $securePassword = ConvertTo-SecureString $password -AsPlainText -Force
-    $serviceCredential = New-Object System.Management.Automation.PSCredential ("$ENV:COMPUTERNAME\$username",
$securePassword)
 
     ###
     ### Stop all services before proceeding with the install step, otherwise
@@ -105,14 +107,23 @@ function Main( $scriptDir )
     ###
     ### Install and Configure Core
     ###
+    # strip out domain/machinename if it exists. will not work with domain users.
+    $shortUsername = $username
+    if($username.IndexOf('\') -ge 0)
+    {
+        $shortUsername = $username.SubString($username.IndexOf('\') + 1)
+    }
+
     Install "Core" $NodeInstallRoot $serviceCredential ""
     Configure "Core" $NodeInstallRoot $serviceCredential @{
         "fs.checkpoint.dir" = "$ENV:HDFS_DATA_DIR\2nn";
-        "fs.checkpoint.edits.dir" = "$ENV:HDFS_DATA_DIR\2nn"}
+        "fs.checkpoint.edits.dir" = "$ENV:HDFS_DATA_DIR\2nn";
+        "hadoop.proxyuser.$shortUsername.groups" = "HadoopUsers";
+        "hadoop.proxyuser.$shortUsername.hosts" = "localhost";}
 
     ###
     ### Install and Configure HDFS
-    ###
+    ###    
     Install "Hdfs" $NodeInstallRoot $serviceCredential $hdfsRoles
     Configure "Hdfs" $NodeInstallRoot $serviceCredential @{
         "dfs.name.dir" = "$ENV:HDFS_DATA_DIR\nn";

Modified: hadoop/common/branches/branch-1-win/src/packages/win/template/conf/log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/packages/win/template/conf/log4j.properties?rev=1452668&r1=1452667&r2=1452668&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/packages/win/template/conf/log4j.properties (original)
+++ hadoop/common/branches/branch-1-win/src/packages/win/template/conf/log4j.properties Tue
Mar  5 04:54:55 2013
@@ -132,7 +132,7 @@ log4j.appender.MRAUDIT.DatePattern=.yyyy
 
 # Custom Logging levels
 
-hadoop.metrics.log.level=INFO
+hadoop.metrics.log.level=WARN
 #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
 #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
 #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG



Mime
View raw message