hadoop使用简介

news2024/11/18 3:43:43

git clone

hadoop源码地址：https://gitee.com/CHNnoodle/hadoop.git
git clone错误：

Filename too long错误，使用git config --global core.longpaths true
git clone https://gitee.com/CHNnoodle/hadoop.git -b rel/release-3.2.2 拉取指定tag版本

hadoop安装包地址：https://mirrors.cloud.tencent.com/apache/hadoop/common/

windows平台下载，https://github.com/cdarlint/winutils替换hadoop对应版本的bin文件

hadoop

core-site.xml，hadoop中间文件的输出路径

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
      <name>hadoop.tmp.dir</name>
      <value>/D:/ruanjian/hadoop/hadoop-3.3.5/data</value>    
    </property>
    <property>
      <name>fs.defaultFS</name>
      <value>hdfs://localhost:9000</value>
    </property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/D:/ruanjian/hadoop/hadoop-3.3.5/data/namenode</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/D:/ruanjian/hadoop/hadoop-3.3.5/data/datanode</value>
    </property>
</configuration>

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
     <property>
            <name>mapreduce.framework.name</name>
             <value>yarn</value>
    </property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<configuration>
     <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
        <value>org.apache.hahoop.mapred.ShuffleHandler</value>
    </property>
</configuration>