flume-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hshreedha...@apache.org
Subject flume git commit: FLUME-1334. Windows startup scripts.
Date Sat, 31 Jan 2015 00:56:46 GMT
Repository: flume
Updated Branches:
  refs/heads/trunk 1d49ef704 -> 62a4cad22


FLUME-1334. Windows startup scripts.

(Roshan Naik via Hari)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/62a4cad2
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/62a4cad2
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/62a4cad2

Branch: refs/heads/trunk
Commit: 62a4cad223aebfa0aa0f9d20c1c37f04a8b918a9
Parents: 1d49ef7
Author: Hari Shreedharan <hshreedharan@apache.org>
Authored: Fri Jan 30 16:55:47 2015 -0800
Committer: Hari Shreedharan <hshreedharan@apache.org>
Committed: Fri Jan 30 16:55:47 2015 -0800

----------------------------------------------------------------------
 bin/flume-ng.cmd |  19 +++
 bin/flume-ng.ps1 | 403 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 422 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/62a4cad2/bin/flume-ng.cmd
----------------------------------------------------------------------
diff --git a/bin/flume-ng.cmd b/bin/flume-ng.cmd
new file mode 100644
index 0000000..b1b97a9
--- /dev/null
+++ b/bin/flume-ng.cmd
@@ -0,0 +1,19 @@
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM   http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+
+powershell.exe -NoProfile -InputFormat none -ExecutionPolicy unrestricted -File %~dp0flume-ng.ps1
%*
+

http://git-wip-us.apache.org/repos/asf/flume/blob/62a4cad2/bin/flume-ng.ps1
----------------------------------------------------------------------
diff --git a/bin/flume-ng.ps1 b/bin/flume-ng.ps1
new file mode 100755
index 0000000..1638bf1
--- /dev/null
+++ b/bin/flume-ng.ps1
@@ -0,0 +1,403 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+param(
+    [Parameter(Position=0, Mandatory=$true)]
+    [ValidateSet('help','agent','avro-client','version')] $command,
+    [Parameter()] [string] $conf,
+    [Parameter()] [Alias('C')] [string] $classPath,
+    [Parameter()] [Alias('property')] [string] $javaProperties,
+    [Parameter()] [Alias('d')] [switch] $dryrun,
+    [Parameter()] [Alias('plugins-path')] [string] $pluginsPath,
+
+    [Parameter()] [Alias('f',"conf-file")] [string] $confFile,
+    [Parameter()] [Alias('n')] [string] $name,
+
+    [Parameter()] [string] $rpcProps,
+    [Parameter()] [Alias('H',"host")] [string] $avroHost,
+    [Parameter()] [Alias('p',"port")] [string] $avroPort,
+    [Parameter()] [Alias("dirname")]  [string] $avroDirName,
+    [Parameter()] [Alias("filename")] [string] $avroFileName,
+    [Parameter()] [Alias('R',"headerFile")] [string] $avroHeaderFile
+)
+
+# when invoked from cmd shell, array arguments are treated as array with one string value
+# so we accept classpath and javaProperty as string argument and split explicitly into array
+
+if ($classPath -ne "") {
+   $classPathArr = $classPath.Split(";")
+} else {
+   $classPathArr = @()
+}
+
+if($javaProperties -ne "") {
+   $javaPropertiesArr = $javaProperties.Split(";")  # properties may have embedded comma,
so using ; as delim
+} else {
+   $javaPropertiesArr = @()
+}
+
+# FUNCTIONS
+Function PrintUsage {
+    Write-Host "
+Usage: flume-ng <command> [options]...
+
+commands:
+  help                  display this help text
+  agent                 run a Flume agent
+  avro-client           run an avro Flume client
+  version               show Flume version info
+
+global options:
+  -conf  <conf>                           use configs in <conf> directory
+  -classpath,-C  ""value1;value2; ..""    append to the classpath
+  -property ""name1=val;name2=val; ..""   sets a JDK system property value
+  -dryrun,-d                              do not actually start Flume (test)
+  -plugins-path ""dir1;dir2""             semi-colon-separated list of plugins.d directories.
See the
+                                          plugins.d section in the user guide for more details.
+                                          Default: \$FLUME_HOME/plugins.d
+
+agent options:
+  -conf-file,-f  <file>                specify a config file (required)
+  -name,-n  <name>                     the name of this agent (required)
+
+avro-client options:
+  -rpcProps <file>           RPC client properties file with server connection params
+  -host,-H  <host>           hostname to which events will be sent (required)
+  -port,-p  <port>           port of the avro source (required)
+  -dirname <dir>             directory to stream to avro source
+  -filename  <file>          text file to stream to avro source [default: std input]
+  -headerFile,-R <file>      headerFile containing headers as key/value pairs on each
new line
+
+  Either --rpcProps or both --host and --port must be specified.
+
+Note that if <conf> directory is specified, then it is always included first
+in the classpath."
+}
+
+Function EnumerateJars($path) {
+   return Get-ChildItem $path -Filter *.jar | ? { "$_" -notmatch "slf4j-(api|log4j12).*jar"
} | % { $_.FullName}
+}
+
+Function GetHadoopHome {
+    if($env:HADOOP_PREFIX) {
+        $hadoopBin = "$env:HADOOP_PREFIX\bin;"
+    } elseif ($env:HADOOP_HOME) {
+        $hadoopBin = "$env:HADOOP_HOME\bin;"
+    }
+
+    #Searches for hadoop.cmd in the HADOOP_HOME, current directory and path
+    [String[]] $hadoopPaths = ("$hadoopBin;.;$env:PATH").Split(";") |
+                                 ? { "$_" -ne "" -and (Test-Path $_ )} |
+                                 ? { Test-Path ( Join-Path $_ "hadoop.cmd" ) }
+    if($hadoopPaths -ne $Null ) {
+       $binDir = $hadoopPaths[0] + "\.."
+       return  Resolve-Path $binDir
+    }
+    Write-Host "WARN: HADOOP_PREFIX or HADOOP_HOME not found"
+    return $Null
+}
+
+Function GetHbaseHome() {
+    if($env:HBASE_HOME) {
+        return $env:HBASE_HOME
+    }
+
+    #Searches for hbase.cmd in the HBASE_HOME, current directory and path
+    [String[]] $hbasePaths = (".;$env:PATH").Split(";") |
+                               ? { "$_" -ne "" -and (Test-Path $_) } |
+                               ? { Test-Path (Join-Path $_ "hbase.cmd") }
+
+    if($hbasePaths -ne $Null) {
+        return $hbasePaths[0]
+    }
+    Write-Host "WARN: HBASE_HOME not found"
+    return $Null
+}
+
+Function GetHiveHome() {
+    if($env:HIVE_HOME) {
+        return $env:HIVE_HOME
+    }
+
+    #Searches for hive.cmd in the HIVE_HOME, current directory and path
+    [String[]] $hivePaths = (".;$env:PATH").Split(";") |
+                               ? { "$_" -ne "" -and (Test-Path $_) } |
+                               ? { Test-Path (Join-Path $_ "hive.cmd") }
+
+    if($hivePaths -ne $Null) {
+        return $hivePaths[0]
+    }
+    Write-Host "WARN: HIVE_HOME not found"
+    return $Null
+}
+
+Function GetJavaLibraryPath ($cmd, $flumeLibDir) {
+    $flumeCoreJar = EnumerateJars( $flumeLibDir ) | ? { $_ -match "flume-ng-core.*jar" }
+    $output = & "$cmd" -classpath $flumeCoreJar org.apache.flume.tools.GetJavaProperty
java.library.path
+
+    # look for the line that has the desired property value
+    if( $output -match "(?m)^java\.library\.path=(.*)$" ) {
+      return $Matches[1].split(";") | % { $_ -replace "\\$" , ""}  # trim trailing \ char
+    }
+    return @();
+}
+
+
+Function GetClassPath ($cmd) {
+    $output = & "$cmd" "classpath"
+	return $output;
+}
+
+Function GetJavaPath {
+    if ($env:JAVA_HOME) {
+        return "$env:JAVA_HOME\bin\java.exe" }ss
+    Write-Host "WARN: JAVA_HOME not set"
+    return '"' + (Resolve-Path "java.exe").Path + '"'
+}
+
+
+function runFlume($javaClassPath, $javaLibraryPath, $javaOptions, $class, $javaProcessArgumentList)
 {
+  [string]$javaPath = GetJavaPath
+  [string]$fullJavaCommand = "-classpath $javaClassPath -Djava.library.path=$javaLibraryPath
$javaOptions $class $javaProcessArgumentList"
+  if ($dryrun) {
+    Write-Host 'Dry run mode enabled (will not actually initiate startup)'
+    Write-Host "$javaPath $fullJavaCommand"
+  } else {
+    Write-Host "
+  Running FLUME $command :
+    class: $class
+    arguments: $javaProcessArgumentList
+    "
+
+    $ErrorActionPreference = "Continue"
+    $x = Start-Process $javaPath -ArgumentList "$fullJavaCommand" -Wait -NoNewWindow
+  }
+}
+
+
+# The script will terminate if any steps fail
+$ErrorActionPreference = "Stop"
+
+#SWITCH FOR THE DIFFERENT COMMANDS
+[string] $javaProcessArgumentList
+switch ($command)
+{
+  'help' {
+    PrintUsage
+    return }
+
+  'agent' {
+    if (!$Name) {
+      PrintUsage
+      Write-Host "ERROR: Name parameter missing"
+      return
+    }
+    if (!$ConfFile) {
+      PrintUsage
+      Write-Host "ERROR: ConfFile parameter missing"
+      return
+    }
+    $class='org.apache.flume.node.Application'
+    $confFile = '"' + (Resolve-Path $confFile).Path + '"'
+    $javaProcessArgumentList = "-n $name -f $confFile"  }
+
+  'avro-client' {
+    $class='org.apache.flume.client.avro.AvroCLIClient'
+    if("$rpcProps" -eq "") {
+      if (!$AvroHost) {
+        PrintUsage
+        Write-Host "ERROR: Avro Host parameter missing"
+        return
+      }
+      if (!$AvroPort) {
+        PrintUsage
+        Write-Host "ERROR: Avro Port parameter missing"
+        return
+      }
+      $javaProcessArgumentList = " -H $AvroHost -p $AvroPort"
+    } else {
+      $javaProcessArgumentList += " --rpcProps ""$rpcProps"""
+    }
+
+    if ($avroHeaderFile) {
+      $avroHeaderFile = (Resolve-Path $avroHeaderFile).Path
+      $javaProcessArgumentList += " -R $avroHeaderFile" }
+    if ($avroFileName) {
+      $avroFileName = (Resolve-Path $avroFileName).Path
+      $javaProcessArgumentList += " -F $avroFileName" }
+    if($avroDirName) {
+      $avroDirName = (Resolve-Path $avroDirName).Path
+      $javaProcessArgumentList += " --dirname ""$avroDirName""" } }
+
+  'version' {
+    $class='org.apache.flume.tools.VersionInfo'
+    $javaProcessArgumentList = "" }
+
+  default {
+    PrintUsage
+    Write-Host "ERROR: Invalid command '$command'"
+    return }
+}
+
+$FlumeHome = $null
+if($env:FLUME_HOME) {
+    $FlumeHome = $env:FLUME_HOME
+}  else {
+    $ScriptPath =  Split-Path -Parent $MyInvocation.MyCommand.Path
+    $FlumeHome =  Split-Path -Parent $ScriptPath
+}
+
+########### Source flume-env.ps1 ##############
+
+# allow users to override the default env vars via conf\flume-env.ps1
+if( "$conf" -eq "" ) {
+    if( Test-path ("$FlumeHome\conf") ) {
+      $conf = "$FlumeHome\conf"
+      Write-Host "WARN: Config directory not set. Defaulting to $conf"
+    }
+}
+if ( "$conf" -ne "" )  {
+  Write-Host "Sourcing environment configuration script $conf\flume-env.ps1"
+  if ( Test-path  "$conf\flume-env.ps1" ) {
+     . "$conf\flume-env.ps1"
+  }  else {
+     Write-Host "WARN: Did not find $conf\flume-env.ps1"
+  }
+}  else {
+  Write-Host "WARN: No configuration directory found! Use --conf <dir> to set."
+}
+
+########### Setup JAVA_OPTS ##############
+
+[string]$javaOptions="$JAVA_OPTS"
+foreach ($opt in $javaPropertiesArr) {
+  $javaOptions = "$javaOptions -D$opt"
+}
+
+########### Setup Classpath ###############
+# flume\conf ; flume_home\lib\* ; cmdline ; env.ps1 ; plugins.d ; hadoop.cpath ; hbase.cpath
; hive.cpath
+
+[string]$javaClassPath='"' + $conf + '"'
+[string]$flumeLibJars=""
+[string]$flumeLibDir = Resolve-Path "$FlumeHome\lib"
+
+# Add FlumeHome\lib\* to class path
+$javaClassPath = "$javaClassPath;""$flumeLibDir\*"""
+$flumeLibJars = "$flumeLibDir\*"""
+
+# Add classpath from cmd line & FLUME_CLASSPATH in flume-env.ps1
+if ( $FLUME_CLASSPATH )  {
+  $classPathArr = $FLUME_CLASSPATH.Split(";")
+}
+foreach ($path in $classPathArr) {
+  $fullPath = (Resolve-Path $path).Path
+  $javaClassPath = "$javaClassPath;""$fullPath"""
+}
+
+$javaLibraryPath = ""
+
+# Add plugins.d into classpath and libpath
+if ("$pluginsPath" -eq "") {
+   $pluginsPath = "$FlumeHome\plugins.d"
+}
+
+foreach($plugin in  $pluginsPath.Split(";") )  {
+  if ( Test-path "$plugin" ) {
+    $pluginTmp =
+        ( (Get-ChildItem "$plugin\*\lib")  + (Get-ChildItem "$plugin\*\libext") ) -join "\*"";"""
+    if( "$pluginTmp" -ne "" ) {
+      $javaClassPath="$javaClassPath;""" + $pluginTmp + "\*"";"
+    }
+
+    $javaLibraryPathTmp = (@(Get-ChildItem "$plugin\*\native") -join "\*"";""")
+    if( "$javaLibraryPathTmp" -ne "" )  {
+       $javaLibraryPath=  "$javaLibraryPath""" + "$javaLibraryPathTmp" + "\*"";"
+    }
+  }
+}
+
+# Add Hadoop classpath &  java.library.path
+$hadoopHome = GetHadoopHome
+if("$hadoopHome" -ne "") {
+  $hadoopCmd = "$hadoopHome\bin\hadoop.cmd"
+  if( Test-Path $hadoopCmd ) {
+    Write-Host "Including Hadoop libraries found in ($hadoopHome) for DFS access"
+    $javaClassPath = "$javaClassPath;""$hadoopHome\conf""";
+    foreach ($path in  GetClassPath $hadoopCmd) {
+      $javaClassPath = "$javaClassPath;""$path"""
+    }
+
+    foreach ( $path in GetJavaLibraryPath $hadoopCmd  $flumeLibDir ) {
+       $javaLibraryPath = "$javaLibraryPath""$path"";"
+    }
+  } else {
+    Write-Host "WARN: $hadoopCmd not be found. Unable to include Hadoop's classpath &
java.library.path"
+  }
+} else {
+  Write-Host "WARN: HADOOP_PREFIX not set. Unable to include Hadoop's classpath & java.library.path"
+}
+
+# Add HBase classpath  & java.library.path
+$hbaseHome = GetHbaseHome
+if( "$hbaseHome" -ne "" ) {
+    $hbaseCmd = "$hbaseHome\bin\hbase.cmd"
+    if( Test-Path $hbaseCmd ) {
+      Write-Host "Including HBase libraries found via ($hbaseHome) for HBase access"
+      foreach ( $path in GetClassPath $hbaseCmd ) {
+          $javaClassPath = "$javaClassPath;""$path"""
+      }
+      $javaClassPath = "$javaClassPath;""$hbaseHome\conf"""
+
+      foreach ( $path in GetJavaLibraryPath $hbaseCmd  $flumeLibDir ) {
+          $javaLibraryPath = "$javaLibraryPath""$path"";"
+      }
+    } else {
+        Write-Host "WARN: $hbaseCmd not be found. Unable to include HBase classpath and java.library.path"
+    }
+}
+
+# Add Hive classpath
+$hiveHome = GetHiveHome
+if( "$hiveHome" -ne "" ) {
+    $hiveLib = "$hiveHome\lib"
+    if( Test-Path $hiveLib ) {
+      Write-Host "Including Hive libraries found via ($hiveHome) for Hive access"
+      $javaClassPath = "$javaClassPath;""$hiveLib\*"""
+    } else {
+      Write-Host "WARN: $hiveLib not found. Unable to include Hive into classpath"
+    }
+
+    $hiveConf = "$hiveHome\conf"
+    if( Test-Path $hiveConf ) {
+      Write-Host "Including Hive conf dir ($hiveConf) in classpath for Hive access"
+      $javaClassPath = "$javaClassPath;""$hiveConf"""
+    } else {
+      Write-Host "WARN: $hiveConf not found. Unable to include it into classpath"
+    }
+
+    $hcatLib = "$hiveHome\hcatalog\share\hcatalog"
+    if( Test-Path $hcatLib ) {
+    Write-Host "Including HCatalog libraries ($hcatLib) for Hive access"
+      $javaClassPath = "$javaClassPath;""$hcatLib\*"""
+    } else {
+        Write-Host "WARN: $hcatLib not found. Unable to include HCatalog into classpath"
+    }
+}
+
+runFlume $javaClassPath $javaLibraryPath $javaOptions $class $javaProcessArgumentList


Mime
View raw message