分享一些日常开发中使用的脚本及工具
#!/user/bin/env python
# coding:utf-8

"""Hadoop 集群安装脚本
@Version    :   1.0.0
@Author     :   Jack

安装 Hadoop 集群

版本 hadoop-3.1.3
"""

import os

from fabric import Connection

hosts = [
    {
        'dhcp_ip': '192.168.10.134',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.135',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.136',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
]

# 构建 Hadoop 集群
for host in hosts:
    # 创建连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    # hadoop 下载地址
    hadoop_download_url = 'https://github.com/jackhawks/dev-helper/releases/download/v1.0.0/hadoop-3.1.3.tar.gz'

    # 下载 hadoop 到 /opt/software 目录下
    conn.sudo(f"wget -P /opt/software {hadoop_download_url}")

    # 解压 hadoop 压缩包到 /opt/module 目录下
    hadoop_file_name = os.path.basename(hadoop_download_url)
    conn.sudo('mkdir -p /opt/module/hadoop')
    conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/module/hadoop")
    conn.run(f"tar -zxvf /opt/software/{hadoop_file_name} --strip-component=1 -C /opt/module/hadoop")

    # 修改 env 环境变量文件
    my_env_path = '/etc/profile.d/my_env.sh'

    hadoop_env = f"""\
    # HADOOP_HOME
    export HADOOP_HOME=/opt/module/hadoop
    export PATH=\\$PATH:\\$HADOOP_HOME/bin
    export PATH=\\$PATH:\\$HADOOP_HOME/sbin\
    """.replace('    ', '')

    conn.run(f"sudo bash -c 'echo \"{hadoop_env}\" >> {my_env_path}'")

    # 使环境变量配置生效
    conn.run('source /etc/profile')

    # 关闭连接
    conn.close()

# 修改 hadoop 集群配置
for host in hosts:
    # 创建连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    name_node = host['hosts'][0]['hostname']
    name_node2 = host['hosts'][-1]['hostname']
    resource_manager = host['hosts'][1]['hostname']

    # 配置 hadoop core-site.xml
    core_site = f"""
    <!-- 指定 NameNode 的地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://{name_node}:8020</value>
    </property>

    <!-- 指定 hadoop 数据的存储目录 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/module/hadoop/data</value>
    </property>

    <!-- 配置 HDFS 网页登录使用的静态用户 -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>{host['general_user']}</value>
    </property>
    """.replace('\n', '\\n')

    conn.run(f"sed -i '/<configuration>/a\\{core_site}' $HADOOP_HOME/etc/hadoop/core-site.xml")

    # 配置 hadoop hdfs-site.xml
    hdfs_site = f"""
	<!-- NN web端访问地址 -->
	<property>
        <name>dfs.namenode.http-address</name>
        <value>{name_node}:9870</value>
    </property>

	<!-- 2NN web端访问地址 -->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>{name_node2}:9868</value>
    </property>
    """.replace('\n', '\\n')

    conn.run(f"sed -i '/<configuration>/a\\{hdfs_site}' $HADOOP_HOME/etc/hadoop/hdfs-site.xml")

    # 配置 hadoop yarn-site.xml
    yarn_site = f"""
    <!-- 指定 Map Reduce 走 shuffle -->
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>

    <!-- 指定 ResourceManager 地址-->
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>{resource_manager}</value>
    </property>

    <!-- 环境变量的继承 -->
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>

    <!-- 开启日志聚集功能 -->
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>

    <!-- 设置日志聚集服务器地址 -->
    <property>  
        <name>yarn.log.server.url</name>  
        <value>http://{name_node}:19888/jobhistory/logs</value>
    </property>

    <!-- 设置日志保留时间为 7 天 -->
    <property>
        <name>yarn.log-aggregation.retain-seconds</name>
        <value>604800</value>
    </property>
    """.replace('\n', '\\n')

    conn.run(f"sed -i '/<configuration>/a\\{yarn_site}' $HADOOP_HOME/etc/hadoop/yarn-site.xml")

    # 配置 mapred-site.xml
    mapred_site = f"""
    <!-- 指定 Map Reduce 程序运行在 Yarn 上 -->
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>

    <!-- 历史服务器端地址 -->
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>{name_node}:10020</value>
    </property>

    <!-- 历史服务器 web 端地址 -->
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>{name_node}:19888</value>
    </property>
    """.replace('\n', '\\n')

    conn.run(f"sed -i '/<configuration>/a\\{mapred_site}' $HADOOP_HOME/etc/hadoop/mapred-site.xml")

    # 配置 workers
    conn.run('> $HADOOP_HOME/etc/hadoop/workers')
    for host in host['hosts']:
        conn.run(f"echo '{host['hostname']}' >> $HADOOP_HOME/etc/hadoop/workers")

    # 关闭连接
    conn.close()

# 上传基础脚本
for host in hosts:
    conn = Connection(host['static_ip'], user=host['root_user'],
                      connect_kwargs={'password': host['root_password']})

    # 上传 hadoop 集群脚本
    hadoop_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'shell-scripts', 'bigdata',
                               'hadoop_cluster.sh')
    conn.put(hadoop_path, '/bin')
    conn.run('chmod +x /bin/hadoop_cluster.sh')

    # 关闭连接
    conn.close()

# 启动 Hadoop 集群
for host in hosts:

    if host['hostname'] == 'hadoop101':
        conn = Connection(host['static_ip'], user=host['general_user'],
                          connect_kwargs={'password': host['general_password']})

        # 第一次启动，需要初始化集群
        conn.run('hdfs namenode -format')

        # 启动 hdfs
        conn.run('$HADOOP_HOME/sbin/start-dfs.sh')

        # 启动服务器
        conn.run('mapred --daemon start historyserver')

        # 关闭连接
        conn.close()

    # 启动 yarn
    if host['hostname'] == 'hadoop102':
        conn = Connection(host['static_ip'], user=host['general_user'],
                          connect_kwargs={'password': host['general_password']})

        conn.run('$HADOOP_HOME/sbin/start-yarn.sh')

        # 关闭连接
        conn.close()

python-scripts/hadoop_cluster_install.py
jackhawks/dev-helper
#!/user/bin/env python
# coding:utf-8

import os

from fabric import Connection

"""HBase 集群环境批量安装脚本
@Version    :   1.0.0
@Author     :   Jack
"""

hosts = [
    {
        'dhcp_ip': '192.168.10.134',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.135',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.136',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
]

for index, host in enumerate(hosts):
    # 建立连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    # hbase 下载地址
    hbase_download_url = 'https://github.com/jackhawks/dev-helper/releases/download/v1.0.0/hbase-2.4.11-bin.tar.gz'

    # 下载 hbase 到 /opt/software 目录下
    conn.sudo(f"wget -P /opt/software {hbase_download_url}")

    # 解压 hbase 压缩包到 /opt/module 目录下
    hbase_file_name = os.path.basename(hbase_download_url)
    conn.sudo('mkdir -p /opt/module/hbase')
    conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/module/hbase")
    conn.run(f"tar -zxvf /opt/software/{hbase_file_name} --strip-component=1 -C /opt/module/hbase")

    # 修改 env 环境变量文件
    my_env_path = '/etc/profile.d/my_env.sh'

    hbase_env = f"""\
    # HBASE_HOME
    export HBASE_HOME=/opt/module/hbase
    export PATH=\\$PATH:\\$HBASE_HOME/bin\
    """.replace('    ', '')

    conn.run(f"sudo bash -c 'echo \"{hbase_env}\" >> {my_env_path}'")

    # 使环境变量配置生效
    conn.run('source /etc/profile')

    # 修改 hbase-env.sh
    conn.run(f"sed -i 's/# export HBASE_MANAGES_ZK=true/export HBASE_MANAGES_ZK=true/' $HBASE_HOME/conf/hbase-env.sh")

    # 修改 hbase-site.xml
    conn.run("sed -i '/<configuration>/, /<\/configuration>/d' $HBASE_HOME/conf/hbase-site.xml")

    hosts_array = []
    for line in host['hosts']:
        hosts_array.append(line['hostname'])

    hbase_site = f"""\
    <configuration>
        <property>
            <name>hbase.zookeeper.quorum</name>
            <value>{','.join(hosts_array)}</value>
        </property>

        <property>
            <name>hbase.rootdir</name>
            <value>hdfs://{hosts_array[0]}:8020/hbase</value>
        </property>

        <property>
            <name>hbase.cluster.distributed</name>
            <value>true</value>
        </property>
    </configuration>\
    """.replace('    ', '')
    conn.run(f"echo '{hbase_site}' >> $HBASE_HOME/conf/hbase-site.xml")

    # regionservers
    regionservers = '\n'.join(hosts_array)
    conn.run(f"echo '{regionservers}' > $HBASE_HOME/conf/regionservers")

    # 解决 HBase 和 Hadoop 的 log4j 兼容性问题
    conn.run("mv $HBASE_HOME/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar "
             "$HBASE_HOME/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar.bak")

    # 关闭连接
    conn.close()

# 启动 HBase 集群
for host in hosts:

    if host['hostname'] == 'hadoop101':
        conn = Connection(host['static_ip'], user=host['general_user'],
                          connect_kwargs={'password': host['general_password']})
        # 集群群启
        conn.run(f"$HBASE_HOME/bin/start-hbase.sh")

        # 关闭连接
        conn.close()

python-scripts/hbase_cluster_install.py
jackhawks/dev-helper
#!/user/bin/env python
# coding:utf-8

import os

from fabric import Connection

"""JDK 集群环境批量安装脚本
@Version    :   1.0.0
@Author     :   Jack

基础初始化集群中的所有机器

1.安装JDK
"""

hosts = [
    {
        'dhcp_ip': '192.168.10.134',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.135',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.136',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
]

for host in hosts:
    # 建立连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    # jdk 下载地址
    jdk_download_url = 'https://github.com/jackhawks/dev-helper/releases/download/v1.0.0/jdk-8u212-linux-x64.tar.gz'

    # 判断文件目录是否存在
    if conn.run('test -d /opt/software', warn=True).failed:
        conn.sudo('mkdir -p /opt/software')
        conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/software")

    if conn.run('test -d /opt/module', warn=True).failed:
        conn.sudo('mkdir -p /opt/module')
        conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/module")

    # 下载 jdk 到 /opt/software 目录下
    conn.sudo(f"wget -P /opt/software {jdk_download_url}")

    # 解压 jdk 压缩包到 /opt/module 目录下
    jdk_file_name = os.path.basename(jdk_download_url)
    conn.sudo('mkdir -p /opt/module/jdk')
    conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/module/jdk")
    conn.run(f"tar -zxvf /opt/software/{jdk_file_name} --strip-component=1 -C /opt/module/jdk")

    # 修改 env 环境变量文件
    my_env_path = '/etc/profile.d/my_env.sh'

    java_env = f"""\
    # JAVA_HOME
    export JAVA_HOME=/opt/module/jdk
    export PATH=\\$PATH:\\$JAVA_HOME/bin\
    """.replace('    ', '')

    conn.run(f"sudo bash -c 'echo \"{java_env}\" >> {my_env_path}'")

    # 使环境变量配置生效
    conn.run('source /etc/profile')

    # 关闭连接
    conn.close()

python-scripts/jdk_cluster_install.py
jackhawks/dev-helper
#!/user/bin/env python
# coding:utf-8

"""Linux 机器基础初始化脚本
@Version    :   1.0.0
@Author     :   Jack

基础初始化集群中的所有机器

1.安装基础软件
2.关闭防火墙
3.修改主机名
4.添加 hosts 映射
5.配置静态 Ip
"""

from fabric import Connection

hosts = [
    {
        'dhcp_ip': '192.168.10.134',
        'root_user': 'root',
        'root_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.134',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.135',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.136',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.135',
        'root_user': 'root',
        'root_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.134',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.135',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.136',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.136',
        'root_user': 'root',
        'root_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.134',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.135',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.136',
                'hostname': 'hadoop103',
            },
        ],
    },
]

for host in hosts:
    conn = Connection(host['dhcp_ip'], user=host['root_user'], connect_kwargs={'password': host['root_password']})

    # 安装基础软件
    conn.run('yum install -y vim rsync lrzsz wget')

    # 关闭防火墙
    conn.run('systemctl stop firewalld && systemctl disable firewalld.service')

    # 修改主机名
    conn.run(f"echo '{host['hostname']}' > /etc/hostname")

    # 添加 hosts 配置
    for index, line in enumerate(host['hosts']):
        if index == 0:
            conn.run(f"echo >> /etc/hosts")
        conn.run(f"echo '{line['static_ip']} {line['hostname']}' >> /etc/hosts")

    # 修改网络配置
    ipa = conn.run("ip a | grep -E \"^[0-9]+: \" | awk -F': ' '$2~/^e/{print $2}' | sed -n '1p'")
    ifcfg_name = ipa.stdout.strip()
    network_ifcfg_path = f"/etc/sysconfig/network-scripts/ifcfg-{ifcfg_name}"

    conn.run(f"sed -i -e 's/BOOTPROTO.*/BOOTPROTO=\"static\"/' -e 's/ONBOOT.*/ONBOOT=\"yes\"/' {network_ifcfg_path}")

    conn.run(f"""
    echo "IPADDR={host['static_ip']}" >> {network_ifcfg_path}
    echo "NETMASK=255.255.255.0" >> {network_ifcfg_path}
    echo "GATEWAY={host['gateway']}" >> {network_ifcfg_path}
    echo "DNS1=8.8.8.8" >> {network_ifcfg_path}
    echo "DNS2=8.8.4.4" >> {network_ifcfg_path}
    """)

    # 重启
    conn.run('reboot', disown=True)

    # 关闭连接
    conn.close()
python-scripts/linux_basis_init.py
jackhawks/dev-helper
#!/user/bin/env python
# coding:utf-8

"""Linux 集群初始化脚本
@Version    :   1.0.0
@Author     :   Jack

初始化集群中的所有机器

1.安装基础软件
2.关闭防火墙
3.修改主机名
4.添加 hosts 映射
5.配置静态 Ip
6.创建工作目录
7.创建普通用户
8.配置 SSH 免密登录
9.上传基础脚本
"""

import os
import time
import socket
import threading
from invoke import Responder
from fabric import Connection

hosts = [
    {
        'dhcp_ip': '192.168.10.133',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.137',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.138',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
]

for host in hosts:
    conn = Connection(host['dhcp_ip'], user=host['root_user'], connect_kwargs={'password': host['root_password']})

    # 安装基础软件
    conn.run('yum install -y vim rsync lrzsz wget dnf')

    # 关闭防火墙
    conn.run('systemctl stop firewalld && systemctl disable firewalld.service')

    # 修改主机名
    conn.run(f"echo '{host['hostname']}' > /etc/hostname")

    # 添加 hosts 配置
    for index, line in enumerate(host['hosts']):
        if index == 0:
            conn.run(f"echo >> /etc/hosts")
        conn.run(f"echo '{line['static_ip']} {line['hostname']}' >> /etc/hosts")

    # 修改网络配置
    ipa = conn.run("ip a | grep -E \"^[0-9]+: \" | awk -F': ' '$2~/^e/{print $2}' | sed -n '1p'")
    ifcfg_name = ipa.stdout.strip()
    network_ifcfg_path = f"/etc/sysconfig/network-scripts/ifcfg-{ifcfg_name}"

    conn.run(f"sed -i -e 's/BOOTPROTO.*/BOOTPROTO=\"static\"/' -e 's/ONBOOT.*/ONBOOT=\"yes\"/' {network_ifcfg_path}")

    conn.run(f"""
    echo "IPADDR={host['static_ip']}" >> {network_ifcfg_path}
    echo "NETMASK=255.255.255.0" >> {network_ifcfg_path}
    echo "GATEWAY={host['gateway']}" >> {network_ifcfg_path}
    echo "DNS1=8.8.8.8" >> {network_ifcfg_path}
    echo "DNS2=8.8.4.4" >> {network_ifcfg_path}
    """)

    # 创建工作目录
    conn.run('mkdir -p /opt/module && mkdir -p /opt/software')

    # 创建普通用户
    conn.run(f"""
    useradd {host['general_user']}
    echo "{host['general_user']}:{host['general_password']}" | chpasswd
    sed -i "/^%wheel/a {host['general_user']}    ALL=(ALL)       NOPASSWD:ALL" /etc/sudoers
    """)

    # 更改工作目录所属用户
    conn.run(f"""
    chown -R {host['general_user']}:{host['general_user']} /opt/module
    chown -R {host['general_user']}:{host['general_user']} /opt/software
    """)

    # 重启
    conn.run('reboot', disown=True)

    # 关闭连接
    conn.close()


def check_connectivity(ip):
    """检查连接性"""

    while True:
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            # 创建一个TCP套接字
            # 设置连接超时时间为1秒
            sock.settimeout(1)
            # 尝试连接目标IP的某个端口（这里选择80端口作为示例）
            sock.connect((ip, 22))
            # 连接成功，输出提示信息并结束脚本
            print(f"IP {ip} is reachable.")
            break
        except socket.error as e:
            # 连接失败，输出错误信息，并等待1秒后再进行下一次尝试
            print(f"Connection to IP {ip} failed: {str(e)}")
            time.sleep(1)
        finally:
            sock.close()


# SSH 免密登录配置-密钥生成
for host in hosts:
    # 启动线程，等待条件满足
    t = threading.Thread(target=check_connectivity, args=(host['static_ip'],))
    t.start()

    # 阻塞，直到条件满足
    t.join()

    # 条件满足，执行操作
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    key_pass = Responder(
        pattern=r'Enter file in which to save the key',
        response='\n',
    )
    passphrase_pass = Responder(
        pattern=r'Enter passphrase',
        response='\n',
    )
    same_passphrase_pass = Responder(
        pattern=r'Enter same passphrase again',
        response='\n',
    )

    conn.run('ssh-keygen -t rsa', pty=True, watchers=[key_pass, passphrase_pass, same_passphrase_pass])

    # 关闭连接
    conn.close()

# SSH 免密登录配置-密钥拷贝
for host in hosts:
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    continue_connecting_pass = Responder(
        pattern=r'\(yes/no\)',
        response='yes\n',
    )

    general_password_pass = Responder(
        pattern=r'password',
        response='{}\n'.format(host['general_password']),
    )

    for line in host['hosts']:
        conn.run(f"ssh-copy-id {line['hostname']}", pty=True,
                 watchers=[continue_connecting_pass, general_password_pass])

    # 关闭连接
    conn.close()

# 上传基础脚本
for host in hosts:
    conn = Connection(host['static_ip'], user=host['root_user'],
                      connect_kwargs={'password': host['root_password']})

    # 上传 jps all
    jps_all_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'shell-scripts', 'basis', 'jpsall')
    conn.put(jps_all_path, '/bin')
    conn.run('chmod +x /bin/jpsall')

    # 上传 xcall
    xcall_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'shell-scripts', 'basis', 'xcall')
    conn.put(xcall_path, '/bin')
    conn.run('chmod +x /bin/xcall')

    # 上传 xsync
    xsync_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'shell-scripts', 'basis', 'xsync')
    conn.put(xsync_path, '/bin')
    conn.run('chmod +x /bin/xsync')

    # 关闭连接
    conn.close()

python-scripts/linux_cluster_init.py
jackhawks/dev-helper
#!/user/bin/env python
# coding:utf-8

import os

from fabric import Connection

"""ZK 集群环境批量安装脚本
@Version    :   1.0.0
@Author     :   Jack
"""

hosts = [
    {
        'dhcp_ip': '192.168.10.134',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop101',
        'static_ip': '192.168.10.101',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.135',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop102',
        'static_ip': '192.168.10.102',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
    {
        'dhcp_ip': '192.168.10.136',
        'root_user': 'root',
        'root_password': '123456',
        'general_user': 'jack',
        'general_password': '123456',
        'hostname': 'hadoop103',
        'static_ip': '192.168.10.103',
        'gateway': '192.168.10.2',
        'hosts': [
            {
                'static_ip': '192.168.10.101',
                'hostname': 'hadoop101',
            },
            {
                'static_ip': '192.168.10.102',
                'hostname': 'hadoop102',
            },
            {
                'static_ip': '192.168.10.103',
                'hostname': 'hadoop103',
            },
        ],
    },
]

for index, host in enumerate(hosts):
    # 建立连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    # zk 下载地址
    zk_download_url = 'https://github.com/jackhawks/dev-helper/releases/download/v1.0.0/apache-zookeeper-3.5.7-bin.tar.gz'

    # 下载 zk 到 /opt/software 目录下
    conn.sudo(f"wget -P /opt/software {zk_download_url}")

    # 解压 zk 压缩包到 /opt/module 目录下
    zk_file_name = os.path.basename(zk_download_url)
    conn.sudo('mkdir -p /opt/module/zookeeper')
    conn.sudo(f"chown -R {host['general_user']}:{host['general_user']} /opt/module/zookeeper")
    conn.run(f"tar -zxvf /opt/software/{zk_file_name} --strip-component=1 -C /opt/module/zookeeper")

    # 创建 zkData 目录, 添加 myid 文件
    conn.run('mkdir -p /opt/module/zookeeper/zkData')
    conn.run(f"echo {index + 1} > /opt/module/zookeeper/zkData/myid")

    # 重命名 zoo.cfg 文件
    conf = '/opt/module/zookeeper/conf'
    conn.run(f"mv {conf}/zoo_sample.cfg {conf}/zoo.cfg")

    # 修改数据存储路径配置
    conn.run(f"sed -i 's#dataDir=.*#dataDir=/opt/module/zookeeper/zkData#' {conf}/zoo.cfg")

    # 添加集群配置
    conn.run(f"echo '####################### zookeeper cluster ##########################' >> {conf}/zoo.cfg")
    for idx, line in enumerate(host['hosts']):
        conn.run(f"echo 'server.{idx + 1}={line['hostname']}:2888:3888' >> {conf}/zoo.cfg")

    # 关闭连接
    conn.close()

# 启动 zk 集群
for index, host in enumerate(hosts):
    # 建立连接
    conn = Connection(host['static_ip'], user=host['general_user'],
                      connect_kwargs={'password': host['general_password']})

    # 启动 zk
    conn.run('/opt/module/zookeeper/bin/zkServer.sh start')

    # 关闭连接
    conn.close()

# 上传基础脚本
for host in hosts:
    conn = Connection(host['static_ip'], user=host['root_user'],
                      connect_kwargs={'password': host['root_password']})

    # 上传 zk 集群脚本
    zk_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'shell-scripts', 'bigdata',
                           'zk_cluster.sh')
    conn.put(zk_path, '/bin')
    conn.run('chmod +x /bin/zk_cluster.sh')

    # 关闭连接
    conn.close()

python-scripts/zk_cluster_install.py
jackhawks/dev-helper