使用Python在Ubuntu14.04上部署Hadoop2.7.7_hadoop2.7.7对应的python hdfs库版本

作者：Gausst松鼠会 | 2024-06-13 09:18:01

踩

hadoop2.7.7对应的python hdfs库版本

使用Python在Ubuntu14.04上部署Hadoop

coding：utf-8

import os
import time

hadoop_environment_set = ‘’’#HADOOP VARIABLES START
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export HADOOP_HOME=/usr/local/hadoop
export PATH= $P A T H :$ HADOOP_HOME/bin
export PATH= $P A T H :$ HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME= $HADOOP_HOME export HADOOP_COMMON_HOME=$ HADOOP_HOME
export HADOOP_HDFS_HOME= $HADOOP_HOME export YARN_HOME=$ HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR= $HADOOP_HOME/lib/native export HADOOP_OPTS="-Djava.library.path=$ HADOOP_HOME/lib"
export JAVA_LIBRARY_PATH= $HADOOP_HOME/lib/native:$ JAVA_LIBRARY_PATH
#HADOOP VARIABLES END’’’
username = ‘hadoop’

class install_and_set_hadoop():
“”“安装配置Hadoop”""

def update_linux(self):
    '''更新Linux'''
    os.system('sudo apt-get update')

def create_hadoop_user(self):
    # 创建Hadoop用户
    command_list = ['sudo useradd %s' % username,
                    'sudo chown %s /home/%s' % (username, username),
                    'sudo passwd %s' % username,
                    'aaa123',
                    'aaa123',
                    'sudo adduser %s sudo' % username,
                    ]

    for i in command_list:
        os.system(i)

def install_ssh(self):
    command_list = ['sudo apt-get install openssh-server',
                    'Y',
                    "ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa",
                    'cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys'
                    ]
    for i in command_list:
        os.system(i)

def install_JDK(self):
    if os.path.exists("/usr/lib/jvm/java-7-openjdk-amd64"):
        print('The JAVA is exist.')
    else:
        command_list = ['sudo apt-get install openjdk-7-jdk',

                        ]

def download_and_install_hadoop(self):

    with open('./temporary.txt', 'w', encoding='utf-8')as f:
        f.write(hadoop_environment_set)

    command_list = ['wget http://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz'
                    'sudo tar -zxvf hadoop-2.7.7.tar.gz',  # 解压
                    'sudo mv hadoop-2.7.7 /usr/local/hadoop',
                    'sudo chown -R %s:%s /usr/local/hadoop' % (username,username),
                    '/usr/local/hadoop/bin/hadoop',
                    'sudo cat temporary.txt >> ~/.bashrc',
                    'source ~/.bashrc'
                    ]

    for i in command_list:
        os.system(i)

def set_hadoop(self):
    # 1.core-site.xml
    if os.path.exists('/usr/local/hadoop/etc/hadoop/core-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/core-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>fs.default.name</name>\n<value>hdfs://localhost:9000</value>\n</property>\n''')

        with open('/usr/local/hadoop/etc/hadoop/core-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 2.hdfs-site.xml
    os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/namenode')
    os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/datanode')
    if os.path.exists('/usr/local/hadoop/etc/hadoop/hdfs-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/hdfs-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>dfs.namenode.name.dir</name>\n<value>file:/usr/local/hadoop/hadoop_data/hdfs/namenode</value>\n</property>\n<property>\n<name>dfs.datanode.data.dir</name>\n<value>file:/usr/local/hadoop/hadoop_data/hdfs/datanode</value>\n</property>\n<property>\n<name>dfs.replication</name>\n<value>2</value>\n</property> ''')
        with open('/usr/local/hadoop/etc/hadoop/hdfs-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 3.mapred-site.xml
    os.system('sudo cp etc/hadoop/mapred-site.xml.template  etc/hadoop/mapred-site.xml')
    time.sleep(0.1)
    if os.path.exists('/usr/local/hadoop/etc/hadoop/mapred-site.xml'):
        with open('/usr/local/hadoop/etc/hadoop/mapred-site.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>',
                      '''<configuration>\n<property>\n<name>mapreduce.framework.name</name>     \n	<value>yarn</value>\n</property>''')

        with open('/usr/local/hadoop/etc/hadoop/mapred-site.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    # 4.hdfs-yarn.xml
    if os.path.exists('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml'):
        with open('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml', 'r', encoding='utf-8')as f:
            a = f.read()
    if '<property>' not in a:
        a = a.replace('<configuration>', '''<configuration>\n<property>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

yarn.nodemanager.aux-services
mapreduce_shuffle

yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler
‘’’)

        with open('/usr/local/hadoop/etc/hadoop/hdfs-yarn.xml', 'w', encoding='utf-8')as f:
            f.write(a)

    os.system('chown -R %s:%s /usr/local/hadoop/hadoop_data/hdfs/*'%(username))
    os.system('sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode')
    # # 最后：创建NameNode、DataNode目录
    # os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/namenode')
    # os.mkdir('/usr/local/hadoop/hadoop_data/hdfs/datanode')
    # sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode
    # chown -R hadoop:hadoop /usr/local/hadoop/hadoop_data/hdfs/*

def install(self):
    self.update_linux()
    self.create_hadoop_user()
    self.install_ssh()
    self.install_JDK()
    self.download_and_install_hadoop()
    self.set_hadoop()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

install_and_set_hadoop().install()

‘’’
1.要输密码说明权限不够，/usr/local/hadoop已经设置过权限
好像是这两个文件夹的权限设置问题
sudo chown 777 /usr/local/hadoop/hadoop_data/hdfs/datanode
chown -R hadoop:hadoop /usr/local/hadoop/hadoop_data/hdfs/*
然后执行/usr/local/hadoop/sbin/start-all.sh
‘’’

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/Gausst松鼠会/article/detail/711747