hadoop-mapreduce-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Juliano Atanazio <juliano.l...@gmail.com>
Subject Manual Installation: CentOS 7 + SystemD + Unit Files + Hadoop at boot
Date Sat, 25 Jun 2016 16:32:45 GMT
Hi.

I'm a novice in Hadoop.
I'm trying to install Hadoop (single node) manually in CentOS 7 with
SystemD to start Hadoop at boot.
But, when I start the DFS service (systemctl start dfs), it starts and then
seconds later dies...
I have "googled" for days and nothing about SystemD...
Below are the steps I have done (with the Java environment OpenJDK with pre
installed):

Excuse-me for my bad english :(




=================================================================================================
# yum erase NetworkManager{,-{libnm,tui,wifi}}

# groupadd -r hdfs

# useradd -r -g hdfs -d /usr/local/hdfs -s /bin/bash -k /etc/skel -c 'HDFS
System User' -m hdfs

# mkdir /usr/local/hdfs/data

# yum install ssh rsync

# cat << EOF >> ~hdfs/.bashrc

export HADOOP_COMMON_HOME='/usr/local/hdfs/hadoop'
export HADOOP_MAPRED_HOME="\${HADOOP_COMMON_HOME}"
export HADOOP_HDFS_HOME="\${HADOOP_COMMON_HOME}"
export YARN_HOME="\${HADOOP_COMMON_HOME}"
export JAVA_HOME='/usr/local/openjdk'
export JRE_HOME="${JAVA_HOME}/jre"
export
PATH="\${PATH}:\${HADOOP_COMMON_HOME}/bin:\${HADOOP_COMMON_HOME}/sbin:\${JAVA_HOME}/bin"
EOF

# chown -R hdfs: ~hdfs/

# su - hdfs

$ tar xf /usr/src/hadoop-2.7.2.tar.gz

$ mv hadoop-2.7.2/ hadoop/

$ rm -f ${HADOOP_COMMON_HOME}/{{,s}bin,etc/hadoop,libexec}/*.cmd

$ cat << EOF > ${HADOOP_COMMON_HOME}/etc/hadoop/core-site.xml
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://0.0.0.0:9000</value>
        <description>NameNode URI</description>
    </property>
</configuration>
EOF


$ cat << EOF > ${HADOOP_COMMON_HOME}/etc/hadoop/yarn-site.xml
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>hadoop</value>
        <description>The hostname of the ResourceManager</description>
    </property>

    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
        <description>shuffle service for MapReduce</description>
    </property>
</configuration>
EOF


$ cat << EOF > ${HADOOP_COMMON_HOME}/etc/hadoop/hdfs-site.xml
<configuration>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:///usr/local/hdfs/data/data</value>
        <description>DataNode directory for storing data
chunks.</description>
    </property>

    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:///usr/local/hdfs/data/name</value>
        <description>NameNode directory for namespace and transaction logs
storage.</description>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>3</value>
        <description>Number of replication for each chunk.</description>
    </property>

    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
        <description>Enable or disable webhdfs. Defaults to
false</description>
    </property>

</configuration>
EOF

$ cat << EOF > ${HADOOP_COMMON_HOME}/etc/hadoop/mapred-site.xml
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
        <description>Execution framework.</description>
    </property>
</configuration>
EOF

$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

$ chmod 0600 ~/.ssh/authorized_keys

$ ssh localhost 'echo test'
test

$ hdfs namenode -format

$ start-dfs.sh

http://localhost:50070/

$ hdfs dfs -mkdir -p /user/${USER}


$ cat << EOF > /tmp/foo.txt
linha 1
linha 2
linha 3
EOF

$ hdfs dfs -put /tmp/foo.txt /user/hdfs/

$ hdfs dfs -cat /user/hdfs/foo.txt

$ jps


# cat << EOF > /lib/systemd/system/hadoop-namenode.service
[Unit]
Description=DFS
After=syslog.target network.target
DefaultDependencies=true

[Service]
Type=simple
User=hdfs
Group=hdfs
Environment=YARN_HOME=/usr/local/hdfs/hadoop
Environment=HADOOP_HDFS_HOME=/usr/local/hdfs/hadoop
Environment=HADOOP_COMMON_HOME=/usr/local/hdfs/hadoop
Environment=JAVA_HOME=/usr/local/openjdk
Environment=HADOOP_MAPRED_HOME=/usr/local/hdfs/hadoop
OOMScoreAdjust=-1000
ExecStart=/usr/local/hdfs/hadoop/sbin/hadoop-daemon.sh start namenode
ExecStop=/usr/local/hdfs/hadoop/sbin/hadoop-daemon.sh stop namenode
TimeoutSec=300
[Install]
WantedBy=multi-user.target
EOF

systemctl enable dfs

systemctl start dfs

Mime
View raw message