hadoop-yarn-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Suma Shivaprasad (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (YARN-6550) Capture launch_container.sh logs
Date Tue, 01 Aug 2017 19:57:00 GMT

    [ https://issues.apache.org/jira/browse/YARN-6550?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16109638#comment-16109638
] 

Suma Shivaprasad commented on YARN-6550:
----------------------------------------

Example launch_container.sh with the patch

{noformat}
#!/bin/bash

export STDOUT="/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001/prelaunch.out"
export STDERR="/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001/prelaunch.err"
echo "Setting up env variables" 1> >(tee -a $STDOUT >&1)
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/etc/hadoop"}
2> >(tee -a $STDERR >&2)
export YARN_CONTAINER_RUNTIME_TYPE="docker" 2> >(tee -a $STDERR >&2)
export JAVA_HOME=${JAVA_HOME:-"/Library/Java/JavaVirtualMachines/jdk1.8.0_131.jdk/Contents/Home"}
2> >(tee -a $STDERR >&2)
export YARN_CONTAINER_RUNTIME_DOCKER_IMAGE="sequenceiq/hadoop-docker" 2> >(tee -a $STDERR
>&2)
export APP_SUBMIT_TIME_ENV="1501617165715" 2> >(tee -a $STDERR >&2)
export NM_HOST="10.22.16.92" 2> >(tee -a $STDERR >&2)
export LD_LIBRARY_PATH="$PWD:$HADOOP_COMMON_HOME/lib/native" 2> >(tee -a $STDERR >&2)
export HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-"/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT"} 2>
>(tee -a $STDERR >&2)
export LOGNAME="sshivaprasad" 2> >(tee -a $STDERR >&2)
export JVM_PID="$$" 2> >(tee -a $STDERR >&2)
export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-"/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT"}
2> >(tee -a $STDERR >&2)
export PWD="/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/container_1501616662779_0002_01_000001"
2> >(tee -a $STDERR >&2)
export HADOOP_COMMON_HOME=${HADOOP_COMMON_HOME:-"/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT"}
2> >(tee -a $STDERR >&2)
export LOCAL_DIRS="/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002"
2> >(tee -a $STDERR >&2)
export APPLICATION_WEB_PROXY_BASE="/proxy/application_1501616662779_0002" 2> >(tee -a
$STDERR >&2)
export SHELL="/bin/bash" 2> >(tee -a $STDERR >&2)
export NM_HTTP_PORT="8042" 2> >(tee -a $STDERR >&2)
export LOG_DIRS="/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001"
2> >(tee -a $STDERR >&2)
export NM_AUX_SERVICE_mapreduce_shuffle="AAA0+gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
" 2> >(tee -a $STDERR >&2)
export NM_PORT="55559" 2> >(tee -a $STDERR >&2)
export USER="sshivaprasad" 2> >(tee -a $STDERR >&2)
export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-"/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT"} 2>
>(tee -a $STDERR >&2)
export CLASSPATH="$PWD:$HADOOP_CONF_DIR:$HADOOP_COMMON_HOME/share/hadoop/common/*:$HADOOP_COMMON_HOME/share/hadoop/common/lib/*:$HADOOP_HDFS_HOME/share/hadoop/hdfs/*:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*:$HADOOP_YARN_HOME/share/hadoop/yarn/*:$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"
2> >(tee -a $STDERR >&2)
export HADOOP_TOKEN_FILE_LOCATION="/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/container_1501616662779_0002_01_000001/container_tokens"
2> >(tee -a $STDERR >&2)
export LOCAL_USER_DIRS="/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/" 2>
>(tee -a $STDERR >&2)
export HOME="/home/" 2> >(tee -a $STDERR >&2)
export CONTAINER_ID="container_1501616662779_0002_01_000001" 2> >(tee -a $STDERR >&2)
export MALLOC_ARENA_MAX="" 2> >(tee -a $STDERR >&2)
echo "Setting up job resources" 1> >(tee -a $STDOUT >&1)
ln -sf "/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/filecache/11/job.jar"
"job.jar" 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
mkdir -p jobSubmitDir 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
ln -sf "/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/filecache/12/job.split"
"jobSubmitDir/job.split" 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
ln -sf "/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/filecache/13/job.xml"
"job.xml" 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
mkdir -p jobSubmitDir 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
ln -sf "/tmp/hadoop-sshivaprasad/nm-local-dir/usercache/sshivaprasad/appcache/application_1501616662779_0002/filecache/10/job.splitmetainfo"
"jobSubmitDir/job.splitmetainfo" 2> >(tee -a $STDERR >&2)
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
echo "Launching container" 1> >(tee -a $STDOUT >&1)
exec /bin/bash -c "$JAVA_HOME/bin/java -Djava.io.tmpdir=$PWD/tmp -Dlog4j.configuration=container-log4j.properties
-Dyarn.app.container.log.dir=/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001
-Dyarn.app.container.log.filesize=0 -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog
 -Xmx1024m org.apache.hadoop.mapreduce.v2.app.MRAppMaster 1>/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001/stdout
2>/2017-07-25/hadoop-3.0.0-beta1-SNAPSHOT/logs/userlogs/application_1501616662779_0002/container_1501616662779_0002_01_000001/stderr
"
hadoop_shell_errorcode=$?
if [ $hadoop_shell_errorcode -ne 0 ]
then
  exit $hadoop_shell_errorcode
fi
{noformat}

> Capture launch_container.sh logs
> --------------------------------
>
>                 Key: YARN-6550
>                 URL: https://issues.apache.org/jira/browse/YARN-6550
>             Project: Hadoop YARN
>          Issue Type: Sub-task
>            Reporter: Wangda Tan
>            Assignee: Suma Shivaprasad
>         Attachments: YARN-6550.patch
>
>
> launch_container.sh which generated by NM will do a bunch of things (like create link,
etc.) while launch a process. No logs captured until {{exec}} is called. We need capture all
failures of launch_container.sh for easier troubleshooting.



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

---------------------------------------------------------------------
To unsubscribe, e-mail: yarn-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: yarn-issues-help@hadoop.apache.org


Mime
View raw message