并发批量管理500台以上服务器脚本分享(shell版)

    也许很多人认为shell不能并发任务,其实可通过其它一些方式来实现。下面的脚本是我批量快速管理500+服务器脚本,阅读该脚本前建议先看《自动执行远程主机命令expect脚本》、《自动远程拷贝expect脚本》和《getopt:命令行选项、参数处理

    用法:

    Usage: ./multi_main.sh [-h|--help]
                     [-v|-V|--version]
                     [-l|--iplist ... ]
                     [-c|--config ... ]
                     [-t|--sshtimeout ... ]
                     [-T|--fttimeout ... ]
                     [-L|--bwlimit ... ]
                     [-n|--ignore]

    cat config.txt #上传文件和执行命令

    file:::~/scripts/test.sh /root/ push
    com:::./test.sh

    cat iplist.txt  #ip列表

    # Usage:
    #ip port user password [password_2] [password_3] [password_4]
    # Example:
    #192.168.0.100 22 root 123456
    192.168.0.200 22 root 123456
    192.168.0.201 22 root 123456
    ...
    ./multi_main.sh -c config.txt -l iplist.txt #开始执行,可查看result目录下的日志来分析是否执行成功

    脚本如下:

    1. mssh.exp 执行远程服务器命令expect脚本
    2. mscp.exp 向远程服务器上传或下载文件expect脚本(rsync)
    3. thread.sh 向一台服务器发起动作
    4. ckssh.py 检查ssh是否通
    5. multi_main.sh 批量执行,对每台调用thread.sh

    mssh.exp:

    #!/usr/bin/expect --
    
    if { [llength $argv] < 4 } {
            puts "Usage: $argv0 ip user passwd port commands timeout"
            exit 1
    }
    
    match_max 600000
    
    set ipcode [lindex $argv 0]
    set ip [exec dc -e $ipcode]
    set user [lindex $argv 1]
    set passwdcode [lindex $argv 2]
    set passwd [exec dc -e $passwdcode]
    set portcode [lindex $argv 3]
    set port [exec dc -e $portcode]
    set commands [lindex $argv 4]
    set timeoutflag [lindex $argv 5]
    
    set yesnoflag 0
    set timeout $timeoutflag
    
    
    for {} {1} {} {
    # for is only used to retry when "Interrupted system call" occured
    
    spawn /usr/bin/ssh -o GSSAPIAuthentication=no -q -l$user -p$port $ip
    
    expect  {
    
            "assword:" {
                    send "$passwd\r"
                    break;
            }
    
            "yes/no)?" {
                    set yesnoflag 1
                    send "yes\r"
                    break;
            }
    
            "FATAL" {
                    puts "\nCONNECTERROR: $ip occur FATAL ERROR!!!\n"
                    exit 1
            }
    
            timeout {
                    puts "\nCONNECTERROR: $ip Logon timeout!!!\n"
                    exit 1
            }
    
            "No route to host" {
                    puts "\nCONNECTERROR: $ip No route to host!!!\n"
                    exit 1
            }
    
            "Connection Refused" {
                    puts "\nCONNECTERROR: $ip Connection Refused!!!\n"
                    exit 1
            }
    
            "Connection refused" {
                    puts "\nCONNECTERROR: $ip Connection Refused!!!\n"
                    exit 1
            }
    
            "Host key verification failed" {
                    puts "\nCONNECTERROR: $ip Host key verification failed!!!\n"
                    exit 1
            }
    
            "Illegal host key" {
                    puts "\nCONNECTERROR: $ip Illegal host key!!!\n"
                    exit 1
            }
    
            "Connection Timed Out" {
                    puts "\nCONNECTERROR: $ip Logon timeout!!!\n"
                    exit 1
            }
    
            "Interrupted system call" {
                    puts "\n$ip Interrupted system call!!!\n"
            }
    }
    }
    
    if { $yesnoflag == 1 } {
            expect {
                    "assword:" {
                            send "$passwd\r"
                    }
    
                    "yes/no)?" {
                            set yesnoflag 2
                            send "yes\r"
                    }
            }
    }
    
    if { $yesnoflag == 2 } {
            expect {
                    "assword:" {
                            send "$passwd\r"
                    }
            }
    }
    
    expect {
            "]" {send "$commands \r"}
            "assword:" {
                    send "$passwd\r"
                    puts "\nPASSWORDERROR: $ip Password error!!!\n"
                    exit 1
            }
    }
    
    expect {
            "]" {send "sleep 1 \r"}
    }
    
    expect {
            "]" {send "exit\r"}
    }
    
    expect eof {
            puts "OK_SSH: $ip\n"
            exit 0;
    }

    mscp.exp:

    #!/usr/bin/expect --
    
    proc Usage_Exit {self} {
            puts ""
            puts "Usage: $self ip user passwd port sourcefile destdir direction bwlimit timeout"
            puts ""
            puts "       sourcefile: a file or directory to be transferred"
            puts "                   需要拷贝目录时目录名后不要带 /, 否则会拷贝该目录下的所有文件"
            puts "       destdir:    the location that the sourcefile to be put into"
            puts "       direction:  pull or push"
            puts "                   pull: remote -> local"
            puts "                   push: local -> remote"
            puts "       bwlimit:    bandwidth limit, kbit/s, 0 means no limit"
            puts "       timeout:    timeout of expect, s, -1 means no timeout"
            puts ""
            exit 1
    }
    
    if { [llength $argv] < 9 } {
            Usage_Exit $argv0
    }
    
    set ipcode [lindex $argv 0]
    set ip [exec dc -e $ipcode]
    set user [lindex $argv 1]
    set passwduncode [lindex $argv 2]
    set passwd [exec dc -e $passwduncode]
    set portcode [lindex $argv 3]
    set port [exec dc -e $portcode]
    set sourcefile [lindex $argv 4]
    set destdir [lindex $argv 5]
    set direction [lindex $argv 6]
    set bwlimit [lindex $argv 7]
    set timeoutflag [lindex $argv 8]
    
    set yesnoflag 0
    set timeout $timeoutflag
    
    for {} {1} {} {
    # for is only used to retry when "Interrupted system call" occured
    
    if { $direction == "pull" } {
    
            if { $bwlimit > 0 } {
                    spawn rsync -crazP --bwlimit=$bwlimit -e "/usr/bin/ssh -o GSSAPIAuthentication=no -q -l$user -p$port" $ip:$sourcefile $destdir
            } elseif { $bwlimit == 0 } {
                    spawn rsync -crazP -e "/usr/bin/ssh -o GSSAPIAuthentication=no -q -l$user -p$port" $ip:$sourcefile $destdir
            } else {
                    Usage_Exit $argv0
            }
    
    } elseif { $direction == "push" } {
    
            if { $bwlimit > 0 } {
                    spawn rsync -crazP --bwlimit=$bwlimit -e "/usr/bin/ssh -o GSSAPIAuthentication=no -q -l$user -p$port" $sourcefile $ip:$destdir
            } elseif { $bwlimit == 0 } {
                    spawn rsync -crazP -e "/usr/bin/ssh -o GSSAPIAuthentication=no -q -l$user -p$port" $sourcefile $ip:$destdir
            } else {
                    Usage_Exit $argv0
            }
    
    } else {
            Usage_Exit $argv0
    }
    
    expect  {
    
            "assword:" {
                    send "$passwd\r"
                    break;
            }
    
            "yes/no)?" {
                    set yesnoflag 1
                    send "yes\r"
                    break;
            }
    
            "FATAL" {
                    puts "\nCONNECTERROR: $ip occur FATAL ERROR!!!\n"
                    exit 1
            }
    
            timeout {
                    puts "\nCONNECTERROR: $ip Logon timeout!!!\n"
                    exit 1
            }
    
            "No route to host" {
                    puts "\nCONNECTERROR: $ip No route to host!!!\n"
                    exit 1
            }
    
            "Connection Refused" {
                    puts "\nCONNECTERROR: $ip Connection Refused!!!\n"
                    exit 1
            }
    
            "Connection refused" {
                    puts "\nCONNECTERROR: $ip Connection Refused!!!\n"
                    exit 1
            }
    
            "Host key verification failed" {
                    puts "\nCONNECTERROR: $ip Host key verification failed!!!\n"
                    exit 1
            }
    
            "Illegal host key" {
                    puts "\nCONNECTERROR: $ip Illegal host key!!!\n"
                    exit 1
            }
    
            "Connection Timed Out" {
                    puts "\nCONNECTERROR: $ip Logon timeout!!!\n"
                    exit 1
            }
    
            "Interrupted system call" {
                    puts "\n$ip Interrupted system call!!!\n"
            }
    }
    
    }
    
    if { $yesnoflag == 1 } {
            expect {
                    "assword:" {
                            send "$passwd\r"
                    }
    
                    "yes/no)?" {
                            set yesnoflag 2
                            send "yes\r"
                    }
            }
    }
    
    if { $yesnoflag == 2 } {
            expect {
                    "assword:" {
                            send "$passwd\r"
                    }
            }
    }
    
    expect {
            "assword:" {
                    send "$passwd\r"
                    puts "\nPASSWORDERROR: $ip Password error!!!\n"
                    exit 1
            }
    
            eof {
                    puts "OK_SCP: $ip\n"
                    exit 0;
            }
    }

    thread.sh:

    #!/bin/bash
    
    # Default Parameters
    myIFS=":::"     # 配置文件中的分隔符
    TOOLDIR=~/scripts
    cd $TOOLDIR
    
    #BEGINDATETIME=`date "+%F %T"`
    
    IP=$1P
    PORT=$2P
    USER=$3
    PASSWD=$4P
    CONFIG_FILE=$5                # 命令列表和文件传送配置列表,关键字为com:::和file:::
    SSHTIMEOUT=$6                 # 远程命令执行相关操作的超时设定,单位为秒
    SCPTIMEOUT=$7                 # 文件传送相关操作的超时设定,单位为秒
    BWLIMIT=$8                    # 文件传送的带宽限速,单位为kbit/s
    
    # 针对一个$IP,执行配置文件中的一整套操作
    while read eachline
    do
            # 必须以com或file开头
            [ -z "`echo $eachline | grep -E '^com|^file'`" ] && continue
    
            myKEYWORD=`echo $eachline | awk -F"$myIFS" '{ print $1 }'`
            myCONFIGLINE=`echo $eachline | awk -F"$myIFS" '{ print $2 }'`
    
            # 配置文件中有关键字file:::,就调用mscp.exp进行文件传送
            if [ "$myKEYWORD"x == "file"x ]; then
                    SOURCEFILE=`echo $myCONFIGLINE | awk '{ print $1 }'`
                    DESTDIR=`echo $myCONFIGLINE | awk '{ print $2 }'`
                    DIRECTION=`echo $myCONFIGLINE | awk '{ print $3 }'`
                    $TOOLDIR/mscp.exp $IP $USER $PASSWD $PORT $SOURCEFILE $DESTDIR $DIRECTION $BWLIMIT $SCPTIMEOUT
    
                    [ $? -ne 0 ] && echo -e "\033[31mSCP Try Out All Password Failed\033[0m\n"
    
            # 配置文件中有关键字com:::,就调用mssh.exp进行远程命令执行
            elif [ "$myKEYWORD"x == "com"x ]; then
                    $TOOLDIR/mssh.exp $IP $USER $PASSWD $PORT "${myCONFIGLINE}" $SSHTIMEOUT
                    #echo  $IP $USER $PASSWD $PORT "${myCONFIGLINE}" $SSHTIMEOUT
                    [ $? -ne 0 ] && echo -e "\033[31mSSH Try Out All Password Failed\033[0m\n"
    
            else
                    echo "ERROR: configuration wrong! [$eachline] "
                    echo "       where KEYWORD should not be [$myKEYWORD], but 'com' or 'file'"
                    echo "       if you dont want to run it, you can comment it with '#'"
                    echo ""
                    exit
            fi
    
    done < $CONFIG_FILE
    
    #ENDDATETIME=`date "+%F %T"`
    
    #echo "$BEGINDATETIME -- $ENDDATETIME"
    #echo "$0 $* --excutes over!"
    
    exit 0

    ckssh.py:

    #!/usr/bin/python
    import socket,sys
    sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sk.settimeout(1)
    try:
        sk.connect((sys.argv[1],int(sys.argv[2])))
        print 'ok'
    except Exception:
        print 'no'
    sk.close()

    multi_main.sh:

    #!/bin/bash
    #Blog: linuxeye.com
    
    ######################  proc defination  ########################
    # ignore rule
    ignore_init()
    {
            # ignore password
            array_ignore_pwd_length=0
            if [ -f ./ignore_pwd ]; then
                    while read IGNORE_PWD
                    do
                            array_ignore_pwd[$array_ignore_pwd_length]=$IGNORE_PWD
                            let array_ignore_pwd_length=$array_ignore_pwd_length+1
                    done < ./ignore_pwd
            fi
    
            # ignore ip address
            array_ignore_ip_length=0
            if [ -f ./ignore_ip ]; then
                    while read IGNORE_IP
                    do
                            array_ignore_ip[$array_ignore_ip_length]=$IGNORE_IP
                            let array_ignore_ip_length=$array_ignore_ip_length+1
                    done < ./ignore_ip
            fi
    }
    
    show_version()
    {
            echo "version: 1.0"
            echo "updated date: 2014-05-28"
    }
    
    show_usage()
    {
            echo -e "`printf %-16s "Usage: $0"` [-h|--help]"
            echo -e "`printf %-16s ` [-v|-V|--version]"
            echo -e "`printf %-16s ` [-l|--iplist ... ]"
            echo -e "`printf %-16s ` [-c|--config ... ]"
            echo -e "`printf %-16s ` [-t|--sshtimeout ... ]"
            echo -e "`printf %-16s ` [-T|--fttimeout ... ]"
            echo -e "`printf %-16s ` [-L|--bwlimit ... ]"
            echo -e "`printf %-16s ` [-n|--ignore]"
            #echo "ignr_flag: 'ignr'-some ip will be ignored; otherwise-all ip will be handled"
    }
    
    TOOLDIR=~/scripts
    cd $TOOLDIR
    
    IPLIST="iplist.txt"                     # IP列表,格式为IP 端口 用户名 密码
    CONFIG_FILE="config.txt"                # 命令列表和文件传送配置列表,关键字为com:::和file:::
    IGNRFLAG="noignr"                       # 如果置为ignr,则脚本会进行忽略条件的判断
    SSHTIMEOUT=100                          # 远程命令执行相关操作的超时设定,单位为秒
    SCPTIMEOUT=2000                         # 文件传送相关操作的超时设定,单位为秒
    BWLIMIT=1024000                         # 文件传送的带宽限速,单位为kbit/s
    [ ! -d "result" ] && mkdir result
    
    # 入口参数分析
    TEMP=`getopt -o hvVl:c:t:T:L:n --long help,version,iplist:,config:,sshtimeout:,fttimeout:,bwlimit:,ignore -- "$@" 2>/dev/null`
    
    [ $? != 0 ] && echo -e "\033[31mERROR: unknown argument! \033[0m\n" && show_usage && exit 1
    
    # 会将符合getopt参数规则的参数摆在前面,其他摆在后面,并在最后面添加--
    eval set -- "$TEMP"
    
    while :
    do
            [ -z "$1" ] && break;
            case "$1" in
                    -h|--help)
                            show_usage; exit 0
                            ;;
                    -v|-V|--version)
                            show_version; exit 0
                            ;;
                    -l|--iplist)
                            IPLIST=$2; shift 2
                            ;;
                    -c|--config)
                            CONFIG_FILE=$2; shift 2
                            ;;
                    -t|--sshtimeout)
                            SSHTIMEOUT=$2; shift 2
                            ;;
                    -T|--fttimeout)
                            SCPTIMEOUT=$2; shift 2
                            ;;
                    -L|--bwlimit)
                            BWLIMIT=$2; shift 2
                            ;;
                    -n|--ignore)
                            IGNRFLAG="ignr"; shift
                            ;;
                    --)
                            shift
                            ;;
                    *)
                            echo -e "\033[31mERROR: unknown argument! \033[0m\n" && show_usage && exit 1
                            ;;
            esac
    done
    
    ################  main  #######################
    BEGINDATETIME=`date "+%F %T"`
    [ ! -f $IPLIST ] && echo -e "\033[31mERROR: iplist \"$IPLIST\" not exists, please check! \033[0m\n" && exit 1
    
    [ ! -f $CONFIG_FILE ] && echo -e "\033[31mERROR: config \"$CONFIG_FILE\" not exists, please check! \033[0m\n" && exit 1
    
    IP_count=$(egrep -v '^#|^$' $IPLIST|wc -l)
    IP_init=1
    while [[ $IP_init -le $IP_count ]]
    do
            egrep -v '^#|^$' $IPLIST | sed -n "$IP_init,$(expr $IP_init + 50)p" > $IPLIST.tmp #并发50
    
            IPSEQ=0
    
            while read IP PORT USER PASSWD PASSWD_2ND PASSWD_3RD PASSWD_4TH OTHERS
           # while read Line
            do
                    [ -z "`echo $IP | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|CNS'`" ] && continue
                    if [ "`python $TOOLDIR/ckssh.py $IP $PORT`" == 'no' ];then
                            [ ! -e ipnologin.txt ] && > ipnologin.txt
                            [ -z "`grep $IP ipnologin.txt | grep $(date +%F)`" ] && echo "`date +%F_%H%M` $IP" >> ipnologin.txt
                            continue
                    fi
    
                    let IPSEQ=$IPSEQ+1
    
                    # 如果启用了忽略,则进入忽略流程
                    if [ $IGNRFLAG == "ignr" ]; then
                            ignore_init
                            ignored_flag=0
    
                            i=0
                            while [ $i -lt $array_ignore_pwd_length ]
                            do
                                    [ ${PASSWD}x == ${array_ignore_pwd[$i]}x ] && ignored_flag=1 && break
                                    let i=$i+1
                            done
    
                            [ $ignored_flag -eq 1 ] && continue
    
                            j=0
                            while [ $j -lt $array_ignore_ip_length ]
                            do
                                    [ ${IP}x == ${array_ignore_ip[$j]}x ] && ignored_flag=1 && break
                                    let j=$j+1
                            done
    
                            [ $ignored_flag -eq 1 ] && continue
                    fi
    
                    ####### Try password from here ####
                    #for PW in $PASSWD $PASSWD_2ND $PASSWD_3RD $PASSWD_4TH
                    #do
                    #        PASSWD_USE=$PW
                    #        $TOOLDIR/ssh.exp $IP $USER $PW $PORT true $SSHTIMEOUT
                    #        [ $? -eq 0 ] && PASSWD_USE=$PW && break
                    #done
                    PASSWD_USE=$PASSWD
    
                    IPcode=$(echo "ibase=16;$(echo "$IP" | xxd -ps -u)"|bc|tr -d '\\'|tr -d '\n')
                    Portcode=$(echo "ibase=16;$(echo "$PORT" | xxd -ps -u)"|bc|tr -d '\\'|tr -d '\n')
                    #USER=$USER
                    PWcode=$(echo "ibase=16;$(echo "$PASSWD_USE" | xxd -ps -u)"|bc|tr -d '\\'|tr -d '\n')
                    Othercode=$(echo "ibase=16;$(echo "$OTHERS" | xxd -ps -u)"|bc|tr -d '\\'|tr -d '\n')
                    #echo $IPcode $Portcode $USER $PWcode $CONFIG_FILE $SSHTIMEOUT $SCPTIMEOUT $BWLIMIT $Othercode
                    ./thread.sh $IPcode $Portcode $USER $PWcode $CONFIG_FILE $SSHTIMEOUT $SCPTIMEOUT $BWLIMIT $Othercode | tee result/$IP.log &
            done < $IPLIST.tmp
            sleep 3
            IP_init=$(expr $IP_init + 50)
    done
    
    ENDDATETIME=`date "+%F %T"`
    
    echo "$BEGINDATETIME -- $ENDDATETIME"
    echo "$0 $* --excutes over!"
    
    exit 0
    Sun Jun 1 17:27:37 CST 2014
    • 本文由 发表于 2014-06-01
    • 转载请务必保留本文链接:https://linuxeye.com/398.html
    shell字符串处理 脚本

    shell字符串处理

    字符串操作(长度,读取,替换) 表达式 含义 ${#string} $string的长度 ${string:position} 在$string中, 从位置$position开始提取子串 ${stri...
    Linux shell用法和技巧 脚本

    Linux shell用法和技巧

    检查远程端口是否对bash开放: echo >/dev/tcp/8.8.8.8/53 && echo "open" 让进程转入后台: Ctrl + z 将进程转到前台: fg 产...
    匿名

    发表评论

    匿名网友

      • 深蓝 深蓝

        暂时只有几十台,观摩一下。

        • 浅蓝 浅蓝

          版主,写的不错 ,但是我怎么执行命令不成功啊。

            • yeho yeho

              @ 浅蓝 什么报错?

                • 浅蓝 浅蓝

                  @ yeho 不存在报错,是我搞错了,是否可以在远程主机上执行本地脚本?

              • 唯影 唯影

                请问版主,你用这个脚本管理500台机器同时执行某个任务,一般需要多长时间

                  • yeho yeho

                    @ 唯影 具体要看并发数和单台服务器执行命令时间。

                      • 唯影 唯影

                        @ yeho 额,像这样,假如500台机器上面都分别有一个10G的数据库,我要同时发出执行备份的命令,请问怎样控制并发数?我看了以上的脚本没找到控制并发数量的参数,希望指点指点

                    • Q丶 Q丶

                      执行这个脚本时都需要什么环境,expect和Python的环境都要配置吗