#!/usr/bin/env bash

function Error() {
  local msg=$1
  echo -e "\033[31m${msg}\033[0m" >&2
}

function Info() {
  local msg=$1
  echo -e "\033[34m${msg}\033[0m"
}

function Warning() {
  local msg=$1
  echo -e "\033[35m${msg}\033[0m" >&2
}

function readp() {
  local msg=$1
  local yes=$2

  if [[ "${STEAMER_INSTALLER_CONFIRM_STEP}x" != "yx" ]]; then
    read -p "${msg}" key
    if [[ ${key} == ${yes} ]]; then
      return 0
    fi
  else
    return 0
  fi
  return 1
}

function readp_loop() {
  local msg=$1
  local yes=$2
  local no=$3

  while true; do
    if [[ "${STEAMER_INSTALLER_CONFIRM_STEP}x" != "yx" ]]; then
      read -p "${msg}" key
      if [[ ${key} == ${yes} ]]; then
        return 0
      fi
      if [[ ${key} == ${no} ]]; then
        return 1
      fi
    fi
  done
  return 1
}

function readp_exit() {
  local msg=$1
  local yes=$2

  readp "${msg}" ${yes}

  if [[ $? -ne 0 ]]; then
    echo "输入错误,退出!"
    exit 1
  fi
}

# 获取参数数组
function get_args_array() {
  start_index=$1
  old_arg_index=0
  new_arg_index=0
  new_args=()
  for arg in "$@"; do
    if [[ ${old_arg_index} -ge ${start_index} ]]; then
      new_args[$new_arg_index]="$arg"
      new_arg_index=$(($new_arg_index + 1))
    fi
    old_arg_index=$(($old_arg_index + 1))
  done

  echo "${new_args[@]}"
}

# 将字符串分隔为数组
function split_string() {
  the_str=$1
  separator=$2

  local IFS="$separator"
  string_array=($the_str)

  echo "${string_array[@]}"
}

# 将数组的元素连接为一个字符串
function join_array() {
  local joiner=$1
  local eles=$(get_args_array 2 "$@")

  local result=''
  local tmp_i=0
  for ele in ${eles}; do
    if [[ ${tmp_i} -ne 0 ]]; then
      result="${result}${joiner}$ele"
    else
      result="$ele"
    fi
    tmp_i=$(($tmp_i + 1))
  done

  echo ${result}
}

# 数组中是否包含指定的值
function array_contains() {
  local the_ele="$1"
  local the_array=$(get_args_array 2 "$@")

  for ele in ${the_array}; do
    if [[ "$ele" == "$the_ele" ]]; then
      echo 'true'
      return 0
    fi
  done

  echo 'false'
}

# 返回两个数组的交集
function array_intersection() {
  local array_a=($1)
  local array_b=($2)
  local array_int=()
  local array_int_index=0

  for ele_a in "${array_a[@]}"; do
    for ele_b in "${array_b[@]}"; do
      if [ "$ele_a" == "$ele_b" ]; then
        array_int[$array_int_index]=$ele_a
        array_int_index=$(($array_int_index + 1))
      fi
    done
  done

  echo "${array_int[@]}"
}

# 返回数组a中不存在数组b的元素集合
function array_difference() {
  local array_a=($1)
  local array_b=($2)
  local array_int=()
  local array_int_index=0

  for ele_a in "${array_a[@]}"; do
    for ele_b in "${array_b[@]}"; do
      if [ "$ele_a" != "$ele_b" ]; then
        array_int[$array_int_index]=$ele_a
        array_int_index=$(($array_int_index + 1))
      fi
    done
  done

  echo "${array_int[@]}"
}

# 获取绝对数
abs() { echo ${1#-}; }

# 拷贝SSH证书到指定节点
function copy_ssh_cert() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")

  for host in ${hosts}; do
    ssh-copy-id ${user}@"$host"
    # 测试登录是否正常
    ssh "$user@$host" "echo 2>&1" && echo "copy ssh cert to [$host] OK" || exit 1
  done
}

# 创建主机列表文件
function create_host_file() {
  local user=$1
  local host_file=$2
  local hosts=$(get_args_array 3 "$@")

  cat /dev/null >${host_file}
  for host in ${hosts}; do
    echo "$user@$host" >>${host_file}
  done
}

# 同步跨主机同步文件
function pscp_files() {
  local host_file=$1
  local from=$2
  local to=$3

  if [[ -n "$STEAMER_INSTALLER_PSCP_COMMAND" ]]; then
    pscp_command=${STEAMER_INSTALLER_PSCP_COMMAND}
  else
    pscp_command=pscp.pssh
  fi

  echo "复制文件: host_file=$host_file, from=$from, to=$to, pscp_command=$pscp_command"

  $pscp_command -h ${host_file} -t 0 -r -p 10 ${from} ${to}
}

function pssh_command() {
  local host_file=$1
  local cmd=$(get_args_array 2 "$@")
  pssh -h ${host_file} -P -t 0 -i "sudo ${cmd}"
}

function ssh_command() {
  local user=$1
  local host=$2
  local cmd=$(get_args_array 3 "$@")
  require_command_success "\"$cmd\"" ssh ${user}@${host} "sudo ${cmd}"
}

function ssh_command_no_sudo() {
  local user=$1
  local host=$2
  local cmd=$(get_args_array 3 "$@")
  require_command_success "\"$cmd\"" ssh ${user}@${host} "${cmd}"
}

function ssh_command_stdout() {
  local user=$1
  local host=$2
  local cmd=$(get_args_array 3 "$@")

  ssh ${user}@${host} "sudo ${cmd}"

  local pass_or_ignore=false
  while ! ${pass_or_ignore}; do
    if [[ "$?" == "0" ]]; then
      pass_or_ignore=true
      # echo "[$service_name] 执行通过"
    else
      readp "执行$the_command 失败,请确认后按Enter重试,或输入 f 跳过该步骤: " "f"
      if [[ $? -eq 0 ]]; then
        pass_or_ignore=true
      fi
    fi
  done
}

function ssh_command_stdout_contraries() {
  local user=$1
  local host=$2
  local cmd=$(get_args_array 3 "$@")

  ssh ${user}@${host} "sudo ${cmd}"

  local pass_or_ignore=false
  while ! ${pass_or_ignore}; do
    if [[ "$?" == "0" ]]; then
      readp "执行$the_command 失败,请确认后按Enter重试,或输入 f 跳过该步骤: " "f"
      if [[ $? -eq 0 ]]; then
        pass_or_ignore=true
      fi
      pass_or_ignore=true
      # echo "[$service_name] 执行通过"
    else
      pass_or_ignore=true

    fi
  done
}

# 检查主机的内存是否足够
function check_host_mem() {
  local user=$1
  local host=$2
  local min_gb=$3

  echo "检测内存: host=$host, min=$min_gb GB"

  local mem_total_gb=$(ssh ${user}@${host} "sudo awk '(\$1 == \"MemTotal:\"){print \$2/1048576}' /proc/meminfo")
  local mem_total_gb=$(echo "$mem_total_gb" | awk '{print int($0)}')
  if [[ ${mem_total_gb} -lt ${min_gb} ]]; then
    readp_exit "主机 [$host] 的可用内存不足,当前$mem_total_gb GB,最低要求$min_gb GB,按Enter退出安装,或输入 f 跳过该校验: " "f"
    #        if [[ "$input_word" != "f" ]]; then
    #            exit 1
    #        fi
  fi
}

# 检查主机的端口是否被占用
function check_host_ports() {
  local user=$1
  local host=$2
  local ports=$(get_args_array 3 "$@")

  echo "检测端口: host=$host, ports=$ports"

  for port in ${ports}; do
    port_conflict=true
    while ${port_conflict}; do
      port_conflict=$(ssh ${user}@${host} "sudo netstat -tlpn | grep \":$port\b\"")
      if [[ -n "$port_conflict" ]]; then
        port_conflict=true
        readp "主机 [$host] 上的端口 [$port] 被占用( netstat -tlpn | grep \":$port\b\" ),请释放端口后按Enter继续,或输入 f 跳过该校验: " "f"
        if [[ $? -eq 0 ]]; then
          port_conflict=false
        fi
      else
        port_conflict=false
      fi
    done
  done
}

# 检查主机指定目录的可用空间是否足够
function check_host_path_remaining_disk_space() {
  local user=$1
  local host=$2
  local path=$3
  local min_gb=$4

  echo "检测磁盘空间: host=$host, path=$path, min=$min_gb GB"

  local space_not_enough=true
  while ${space_not_enough}; do
    remaining_kb=$(ssh -t ${user}@${host} "sudo mkdir -p $path && sudo chown ${user}:${user} $path && df $path | tail -n 1 | awk '{print \$4}'" | col)
    if [[ $? -ne 0 ]]; then
      readp "主机 [$host] 上的目录 [$path] 剩余磁盘空间校验失败,请处理确认后按Enter继续,或输入 f 跳过该校验: " "f"
      if [[ $? -eq 0 ]]; then
        space_not_enough=false
      fi
    else
      remaining_gb=$(($remaining_kb / 1048576))
      if [[ ${remaining_gb} -lt ${min_gb} ]]; then
        readp "主机 [$host] 的安装目录[$path]磁盘空间较小,当前$remaining_gb GB,建议使用大于$min_gb GB安装平台,请释放磁盘空间后按Enter继续,或输入 f 跳过该校验: " "f"
        if [[ $? -eq 0 ]]; then
          space_not_enough=false
        fi
      else
        space_not_enough=false
      fi
    fi
  done
}

# 检查主机指定目录的文件系统
function check_host_path_filesystem() {
  local user=$1
  local host=$2
  local path=$3

  echo "检测文件系统: host=$host, path=$path"

  local fs_invalid=true
  while ${fs_invalid}; do
    local xfs_path=''
    local mounts=$(ssh ${user}@${host} "sudo df --no-sync | awk '{print $6}'")
    local is_head=true
    for mount in ${mounts}; do
      if ${is_head}; then
        is_head=false
      else
        if [[ ${#mount} -gt ${#xfs_path} && "$path" =~ ^$mount.* ]]; then
          xfs_path="$mount"
        fi
      fi
    done
    if [[ -z "$xfs_path" ]]; then
      xfs_path=${path}
    fi

    ssh ${user}@${host} "if [[ \`df -T $xfs_path|grep $xfs_path |head -n 1|awk '{print \$2}'\` == "xfs" ]];then  xfs_info $xfs_path | grep ftype=1 2>&1>/dev/null;fi"
    if [[ $? -ne 0 ]]; then
      readp "主机 [$host] 上的目录 [$path] 文件系统校验( xfs_info $xfs_path | grep ftype=1 )失败,请处理确认后按Enter继续,或输入 f 跳过该校验: " "f"
      if [[ $? -eq 0 ]]; then
        fs_invalid=false
      fi
    else
      fs_invalid=false
    fi
  done
}

# 获取IP对应的网卡设备
function get_network_interface_by_ip() {
  local user=$1
  local host=$2

  # 本地执行
  if [[ -z ${host} ]]; then
    local host=$1
    local network_itf=$(sudo ip route | grep " $host " | awk -F '[ \t*]' '{print $3}' | head -n 1)
  else
    local network_itf=$(ssh ${user}@${host} "sudo ip route | grep ' $host ' | awk -F '[ \t*]' '{print \$3}'")
  fi
  echo ${network_itf}
}

# 获取网卡设备对应的IP
function get_ip_by_interface() {
  local inet=$1
  local ips=$(ip a | grep ${inet} | grep 'inet ' | awk '{print $2}' | awk -F/ '{print $1}')
  if [[ $(echo "$ips" | wc -l) -ne 1 ]]; then Error "存在多个${inet}网卡,请确认"; fi
  ip=$(echo "$ips" | head -n 1)
  echo ${ip}
}

function get_cpu_processor() {
  local user=$1
  local host=$2

  if [[ -z $user ]]; then
    local cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l)
  else
    local cpu_num=$(ssh ${user}@${host} "cat /proc/cpuinfo| grep processor| wc -l")
  fi
  echo ${cpu_num}
}

# 获取默然路由对应的网卡名
function get_default_route_dev() {
  local user=$1
  local host=$2
  #本地执行
  if [[ -z ${user} ]]; then
    dev=$(sudo ip route | grep '^default' | awk -F '[ \t*]' '{print $5}')
  else
    dev=$(ssh ${user}@${host} "sudo ip route | grep '^default' | awk -F '[ \t*]' '{print \$5}'")
  fi
  echo ${dev}
}

function get_mask_by_ip() {
  local user=$1
  local host=$2
  # 本地执行
  if [[ -z $host ]]; then
    local host=$1
    mask=$(ip a | grep " ${host}" | awk '{ ipmask=$2; split(ipmask,mask,"/");print mask[2]}')

  else
    mask=$(ssh ${user}@${host} ip a | grep " ${host}" | awk '{ ipmask=\$2; split(ipmask,mask,"/");print mask[2]}')

  fi
  echo ${mask}
}
# 获取IP对应的网卡设备
function get_network_gateway_by_interface() {
  local user=$1
  local host=$2
  local inet=$3

  # 本地执行 else 远程执行
  if [[ -z ${host} ]]; then
    local inet=$1
    local network_gateway=$(sudo ip route | grep " $inet " | grep '^default' | awk '{print $3}')
  else
    local network_gateway=$(ssh ${user}@${host} "sudo ip route |grep " $inet "|grep '^default'|  awk  '{print \$3}'")
  fi

  echo ${network_gateway}
}

# 检查当前主机与master节点的网络是否通畅
function check_host_net_ping_local() {
  local to_host=$1
  local timeout=$2
  local input_word=""
  local pass_or_ignore=false
  local i=0
  if [[ -z $to_host ]]; then
    Error "ping目标主机不能为空"
    return 1
  fi
  if [[ -z $timeout ]]; then timeout=120; fi

  for i in $(seq $timeout); do
    Info "检测网络ping状态: to=$to_host"
    ret=$(ping -c 10 -i 0.2 -t 255 $to_host 2>&1)
    if [[ $? -eq 0 ]]; then
      return 0
    fi
    Info "检测网络连通性失败!,1秒后重试!"
    sleep 1
  done
  Error "检测网络连通性失败!"
  return 1

  #    require_command_success "主机 [$from_host] 到主机 [$to_host] 的网络连接" ssh ${user}@${from_host} "ping -c 10 -i 0.2 -t 255 $to_host"
}

# 检查主机与master节点的网络是否通畅
function check_host_net_ping() {
  local user=$1
  local from_host=$2
  local to_host=$3
  local input_word=""
  local the_command="ssh ${user}@${from_host} ping -c 10 -i 0.2 -t 255 $to_host"

  local pass_or_ignore=false
  local connect=false
  local no_loss=false
  local no_delay=false

  while ! ${pass_or_ignore}; do
    echo "检测网络ping状态: from=$from_host, to=$to_host"
    ret=$(${the_command})
    if [[ "$?" != "0" ]]; then
      echo -e "\033[31m 检测网络连通性失败 \033[0m"
      readp "执行$the_command: " "f"
      if [[ $? -eq 0 ]]; then
        connect=true
      fi
    else
      connect=true
    fi

    result=$(echo "${ret}" | grep time= | awk '{print $7}' | awk -F= '{print $2}' | awk -F. '{print $1}' | sort -rn)

    local num=$(echo "${result}" | wc -l)
    local max=$(echo "${result}" | head -n 1)

    if [[ ${num} -ne 10 ]]; then
      echo "${ret}"
      echo -e "\033[31m ping 检测出现丢包,会影etcd的数据存储,请处理好网络后输入 f 后继续:\033[0m"
      readp ":" "f"
      if [[ $? -eq 0 ]]; then
        no_delay=true
      fi
    else
      no_delay=true
    fi

    if [[ ${max} -ge 100 ]]; then
      echo "${ret}"
      echo -e "\033[31m ping 检测延迟超过100ms,会影etcd的数据存储,请处理好网络后输入 f 后继续:\033[0m"
      readp ":" "f"
      if [[ $? -eq 0 ]]; then
        no_loss=true
      fi
    else
      no_loss=true
    fi

    if [[ ${connect} == true && ${no_delay} == true && ${no_loss} == true ]]; then
      pass_or_ignore=true
    else
      pass_or_ignore=false
    fi

  done

  #    require_command_success "主机 [$from_host] 到主机 [$to_host] 的网络连接" ssh ${user}@${from_host} "ping -c 10 -i 0.2 -t 255 $to_host"
}

function disk_iops_check_remote() {
  local user=$1
  local host=$2
  local etcd_dir=$3
  local input_word=""
  local pass_or_ignore=false

  # local os_info=$(get_os_info ${user} ${host})
  # if [[ $os_info =~ "Ubuntu" || $os_info =~ "Debian" ]]; then
  #     return 0
  # fi
  while ! ${pass_or_ignore}; do
    ssh ${user}@${host} "sudo mkdir -p ${etcd_dir}"
    ret=$(ssh ${user}@${host} "sudo fio --rw=write --ioengine=sync --fdatasync=1 --directory=${etcd_dir} --size=22m --bs=2300 --name=mytest")
    iops=$(echo "${ret}" | grep iops | awk '{print $4}' | sed 's/,//g')
    fsync=$(echo "${ret}" | grep "99.00th\|fsync" | tail -n 1 | awk -F[\]\[\] '{print $2}' | sed 's/\]//g' | sed 's/,//g')
    local iops_ok=false
    local fsync_ok=false

    if [[ ${iops} -lt 50 ]]; then
      echo "${ret}"
      echo -e "\033[31m min_iops=${iops},小于50(etcd最低要求),会影etcd的数据存储,请处理好后输入 f 后继续:\033[0m"
      readp ":" "f"
      if [[ $? -eq 0 ]]; then
        iops_ok=true
      fi
    else
      iops_ok=true
    fi

    if [[ ${fsync} -gt 10000 ]]; then
      echo "${ret}"
      echo -e "\033[31m fdatasync 99th percentile =${fsync}usec ,大于10ms(etcd最低要求),会影etcd的数据存储,请处理好后输入 f 后继续:\033[0m"
      readp ":" "f"
      if [[ $? -eq 0 ]]; then
        fsync_ok=true
      fi
    else
      fsync_ok=true
    fi

    if [[ ${iops_ok} == true && ${fsync_ok} == true ]]; then
      pass_or_ignore=true
    else
      pass_or_ignore=false
    fi
  done
  Info "iops:[$iops] , fsync:[$fsync]"
}

# 校验主机名是否冲突
function check_hostname_conflict() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")

  echo "检测主机名称配置"

  local host_names=()
  local hostname_index=0
  for host in ${hosts}; do
    hostname_conflict=true
    while ${hostname_conflict}; do
      the_hostname=$(ssh ${user}@${host} sudo hostname)
      if [[ $? -ne 0 ]]; then
        readp_exit "主机 [$host] 的hostname获取失败,请确认处理后按Enter继续"
      else
        hostname_exists=$(array_contains ${the_hostname} ${host_names[@]})
        if ${hostname_exists}; then
          readp "主机 [$host] 的hostname与其他主机冲突,请确认处理后按Enter继续,或输入 f 跳过该校验: " f
          if [[ $? -eq 0 ]]; then
            hostname_conflict=false
          fi
        else
          hostname_conflict=false
          hostname_index=$(($hostname_index + 1))
          host_names[$hostname_index]=${the_hostname}
        fi
      fi
    done
  done
}

function check_default_route() {
  local user=$1
  local host=$2

  local hostname_conflict=true
  while ${hostname_conflict}; do
    defalut_dev=$(get_default_route_dev ${user} ${host})
    if [[ $? -ne 0 ]]; then
      readp "主机 [$host] 未发现默然路由,请添加默然路由后按Enter继续,或输入 f 跳过该校验: " f
      if [[ $? -eq 0 ]]; then
        hostname_conflict=false
        return
      fi
    fi

    ip_dev=$(get_network_interface_by_ip ${user} ${host})

    if [[ "x${defalut_dev}" != "x${ip_dev}" ]]; then
      readp "主机 [$host] 默然路由设备 [${defalut_dev}] 与当前IP设备 [${ip_dev}]不一致,请处理后按Enter继续,或输入 f 跳过该校验: " f
      if [[ $? -eq 0 ]]; then
        hostname_conflict=false
        return
      fi
    else
      hostname_conflict=false
    fi

  done

}

function set_time_from_internet() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")
  for host in ${hosts}; do
    ssh ${user}@${host} "unalias cp ;sudo cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime" 2>&1 >/dev/null
    ssh ${user}@${host} "sudo ntpdate ntp.api.bz"
    ssh ${user}@${host} "sudo hwclock -w"
  done
}

function set_time_from_local() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")
  for host in ${hosts}; do
    ssh ${user}@${host} "sudo unalias cp ;sudo cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime" 2>&1 >/dev/null
    date_str=$(date '+%Y-%m-%d %T' -d +1second)
    ssh ${user}@${host} "sudo date -s \"$date_str\""
    ssh ${user}@${host} "sudo hwclock -w"
  done
}

# 检测时间差值
function check_host_time() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")

  echo "检测时间配置"

  echo "$(timedatectl)"
  echo "如需手动设置时间请执行命令date -s “2019-03-08 10:18:30” && hwclock -w(时间字符串请自行修改为当前时间)"
  readp "本机时间[$(date -R)],请确认时间与时区是否正确,正确请按Enter键继续"

  input_word=$(readp "请选择所有机器校准时间的方式：1：从互联网自动校准.2:自动同步本机时间.3：不做处理(按Enter键继续)")
  case ${input_word} in
  1)
    set_time_from_internet ${user} ${hosts}
    ;;
  2)
    set_time_from_local ${user} ${hosts}
    ;;
  *)
    set_time_from_internet ${user} ${hosts}
    set_time_from_local ${user} ${hosts}
    ;;
  esac

  # 第二次检测
  for host in $hosts; do
    timezone_is_invalid=true
    while $timezone_is_invalid; do
      remoteTZ=$(ssh ${user}@$host "date +%z")
      localTZ=$(date +%z)

      if [ ${remoteTZ} != ${localTZ} ]; then
        readp "主机 [$host] 与本机时区不一致,$localTZ != $remoteTZ，是否自动设置(y/n): " "y"
        if [[ $? -eq 0 ]]; then
          ssh ${user}@${host} "unalias cp ;sudo cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime" 2>&1 >/dev/null
          if [[ $? != 0 ]]; then
            readp "时区校准失败! 请手动校准后按Enter继续"
          fi
        else
          readp "请手动校准时区后按Enter继续"
        fi
      else
        timezone_is_invalid=false
      fi
    done

    time_is_invalid=true
    while ${time_is_invalid}; do
      remote_time=$(ssh ${user}@$host "date +%s")
      local_time=$(date +%s)
      dif=$(abs $(($remote_time - $local_time)))
      if [[ ${dif} -gt 10 ]]; then
        readp "主机 [$host] 与本机时间相差10秒以上,是否自动与公网时间校准(y/n): " "y"
        if [[ $? -eq 0 ]]; then
          ssh ${user}@${host} "ntpdate ntp.api.bz"
          if [[ $? != 0 ]]; then
            readp "时间校准失败! 请手动校准后按Enter继续"
          fi
          ssh ${user}@${host} "sudo hwclock -w"
          if [[ $? != 0 ]]; then
            readp "硬件时间校准失败! 请手动校准后按Enter继续"
          fi
        else
          readp "请手动校准时间后按Enter继续"
        fi
      else
        time_is_invalid=false
      fi
    done
  done
}

# 检测防火墙
function check_host_firewall_and_selinux() {
  local user=$1
  local hosts=$(get_args_array 2 "$@")

  echo "检测主机防火墙配置"

  for host in ${hosts}; do
    firewall_is_valid=true
    while ${firewall_is_valid}; do
      fireStatus=$(ssh ${user}@${host} "sudo systemctl status firewalld |grep 'Active:'")
      fireStatus=$(echo ${fireStatus} | awk '{print $2}')
      if [[ "$fireStatus" != "inactive" ]]; then
        if ${STEAMER_INSTALLER_FORCE_DISABLE_FIREWALL}; then
          #                    input_word="y"
          echo "配置指定强制关闭防火墙!"
        else
          readp "主机 [${host}] 防火墙未关闭，是否自动关闭(y/n): " "y"
        fi
        if [[ $? -eq 0 ]]; then
          echo "自动关闭主机 [${host}] 的防火墙"
          ssh ${user}@${host} "sudo systemctl disable firewalld && sudo systemctl stop firewalld"
          if [[ $? != 0 ]]; then
            readp "自动关闭失败! 请手动关闭后按Enter继续"
          fi
        else
          readp "请手动关闭后按Enter继续,或输入 f 跳过该校验: " "f"
          if [[ $? -eq 0 ]]; then
            firewall_is_valid=false
          fi
        fi
      else
        firewall_is_valid=false
      fi
    done

    selinux_is_valid=true
    ssh ${user}@${host} "which setenforce"
    if [[ $? -ne 0 ]]; then
      selinux_is_valid=false
    fi

    while ${selinux_is_valid}; do
      seStatus=$(ssh ${user}@${host} "sudo cat /etc/selinux/config |grep -v '^#' |grep 'SELINUX='")
      seStatus=$(echo ${seStatus} | awk -F= '{print $2}')
      if [[ "$seStatus" != "disabled" ]]; then
        if ${STEAMER_INSTALLER_FORCE_DISABLE_SELINUX}; then
          #                    input_word="y"
          echo "配置指定强制关闭selinux"
        else
          readp "主机 [${host}] selinux未关闭，是否自动关闭(y/n): " "y"
        fi
        if [[ $? == 0 ]]; then
          echo "自动关闭主机 [${host}] 的selinux"
          ssh ${user}@${host} "sudo setenforce 0 > /dev/null 2>&1 && sudo sed -i -e 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config"
          if [ $? != 0 ]; then
            readp "自动关闭失败! 请手动关闭后按Enter继续"
          fi
        else
          readp "请手动关闭后按Enter继续"
        fi
      else
        selinux_is_valid=false
      fi
    done
  done
}

if [[ -z "$STEAMER_INSTALLER_START_AT_COMPONENT" ]]; then
  __STEAMER_INSTALLER_IGNORE_COMPONENTS=0
else
  __STEAMER_INSTALLER_IGNORE_COMPONENTS=1
fi
function confirm_before_step() {
  local service_name=$1
  local msg=$2
  local key=$3

  if [[ x"$key" == x"yes" ]]; then
    read -p "一般不需要执行此步骤,请输入yes才执行安装 (yes)!" key1
    if [[ x"$key1" == x"yes" ]]; then
      return 0
    fi
    return 1
  fi

  if [[ ${__STEAMER_INSTALLER_IGNORE_COMPONENTS} -eq 1 ]]; then
    if [[ "$service_name" == "$STEAMER_INSTALLER_START_AT_COMPONENT" ]]; then
      __STEAMER_INSTALLER_IGNORE_COMPONENTS=0
    else
      return 1
    fi
  fi
  # 开始安装保存进度
  echo ${service_name} >${CURRENT_HOST_TMP_PATH}/step.temp
  if ${STEAMER_INSTALLER_CONFIRM_BEFORE_STEP}; then
    read -p "即将执行步骤 [$msg],请确认后按Enter继续,或输入 f 跳过该步骤: " input_word
    if [[ "$input_word" == "f" ]]; then
      return 1
    fi
  else
    echo "执行步骤 [$msg]"
  fi
  return 0
}

function confirm_after_step() {
  local msg=$1
  # 安装完成，清理此进度
  echo "" >${CURRENT_HOST_TMP_PATH}/step.temp

  if [ $__STEAMER_INSTALLER_IGNORE_COMPONENTS -eq 1 ]; then
    return 0
  fi

  if $STEAMER_INSTALLER_CONFIRM_AFTER_STEP; then
    read -p "步骤 [$msg] 执行完成,请确认后按Enter继续" input_word
  else
    echo "步骤 [$msg] 执行完成"
  fi
}

function require_command_success() {
  local service_name=$1
  local the_command=$(get_args_array 2 "$@")

  local pass_or_ignore=false
  while ! ${pass_or_ignore}; do
    command_out=$(${the_command[@]})
    if [[ "$?" == "0" ]]; then
      pass_or_ignore=true
      # echo "[$service_name] 执行通过"
    else
      echo "ERROR: $command_out"
      readp "$service_name 执行$the_command 失败,请确认后按Enter重试,或输入 f 跳过该步骤: " "f"
      if [[ $? -eq 0 ]]; then
        pass_or_ignore=true
      fi
    fi
  done
}

function require_command_status() {
  local command_name=$1
  local status_code=$2
  local the_command=$(get_args_array 3 "$@")

  local pass_or_ignore=false
  while ! $pass_or_ignore; do
    ${the_command[@]}
    command_status=$?
    if [[ "$command_status" == "$status_code" ]]; then
      pass_or_ignore=true
      # echo "[$command_name] 执行通过"
    else
      read -p "$command_name 执行失败,请确认后按Enter重试,或输入 f 跳过该步骤: " input_word
      if [[ "$input_word" == "f" ]]; then
        pass_or_ignore=true
      fi
    fi
  done
}

function require_command_success_on_host() {
  local user=$1
  local host=$2
  local command_name=$3
  local the_command=$(get_args_array 4 "$@")

  require_command_success "在主机 [$host] $command_name" ssh ${user}@${host} "sudo sh -c \"${the_command[@]}\""
}

function pod_running() {
  local service_name=$1
  local namespace=$2
  local pod_selector=$3
  local min_pod_replicas=$4
  local pod_containers=$5
  local time_out=$6
  Warning "校验 $service_name 中..."
  if [[ ${time_out} == "" ]]; then time_out=120; fi

  for i in $(seq $time_out); do
    pod_count=$(sudo kubectl get po --no-headers -n $namespace --selector=$pod_selector 2>&1 | grep " $pod_containers/$pod_containers " | grep " Running\|Completed " | wc -l)
    if [[ "$?" == "0" && ${pod_count} -ge ${min_pod_replicas} ]]; then
      Warning "$service_name 校验成功!"
      return 0
    else
      Info "$service_name 校验失败,1秒后重试!"
      sleep 1
    fi
  done
  Error "$service_name 校验失败!"
  exit 1
}

function require_pod_running() {
  local user=$1
  local master_host=$2
  local service_name=$3
  local namespace=$4
  local pod_selector=$5
  local min_pod_replicas=$6
  local pod_containers=$7
  local time_out=$8

  if [[ ${time_out} == "" ]]; then time_out=20; fi

  local pass_or_ignore=false
  local i=0
  while ! ${pass_or_ignore}; do
    sleep 1
    pod_count=$(ssh ${user}@${master_host} "sudo kubectl get po --no-headers -n $namespace --selector=$pod_selector | grep \" $pod_containers/$pod_containers \" | grep \" Running\|Completed \" | wc -l")
    if [[ "$?" == "0" && ${pod_count} -ge ${min_pod_replicas} ]]; then
      pass_or_ignore=true
      echo "$service_name 校验成功!"
      return 0
    else
      if [[ ${i} -ge ${time_out} ]]; then
        readp "$service_name 校验失败,请确认后按Enter重试,或输入 f 跳过该校验: " "f"
        if [[ $? -eq 0 ]]; then
          pass_or_ignore=true
        fi
      fi
    fi
    i=$(($i + 1))
  done
}

function require_container_up() {
  local user=$1
  local target_host=$2
  local service_name=$3
  local keyword=$4
  # 本地执行
  if [[ -z "$service_name" ]]; then
    service_name=$1
    keyword=$2
    timeout=$3
    if [[ -z $timeout ]]; then timeout=120; fi

    for i in $(seq $timeout); do
      command_out=$(docker ps | grep $keyword | grep " Up ")
      if [[ $? -eq 0 ]]; then
        Info "$service_name 校验成功!"
        return 0
      fi
      Info "$service_name 校验失败,1秒后重试!"
      sleep 1
    done
    Error "$service_name 校验失败!"
    return 1
  fi

  #远程执行
  local pass_or_ignore=false
  while ! ${pass_or_ignore}; do
    command_out=$(ssh ${user}@${target_host} "sudo docker ps | grep $keyword | grep \" Up \" || exit 1")
    if [[ "$?" == "0" ]]; then
      pass_or_ignore=true
    else
      readp "$service_name 校验失败,请确认后按Enter重试,或输入 f 跳过该校验: " "f"
      if [[ $? -eq 0 ]]; then
        pass_or_ignore=true
      fi
    fi
  done
}

function get_os_info() {
  local user=$1
  local host=$2

  # 本地执行else 远程执行
  if [[ -z $user ]]; then
    #        echo $(sudo cat /etc/os-release | grep "^NAME" | awk -F= '{print $2}')
    which rpm 2>&1 >/dev/null
    if [[ $? -eq 0 ]]; then
      echo "CentOS"
      return 0
    fi
    which dpkg 2>&1 >/dev/null
    if [[ $? -eq 0 ]]; then
      echo "Ubuntu"
      return 0
    fi
    echo $(sudo cat /etc/os-release | grep "^NAME" | awk -F= '{print $2}')
    return 0
  fi
  echo $(ssh -t ${user}@${host} sudo cat /etc/os-release | grep "^NAME" | awk -F= '{print \$2}')
  return 0
}

function get_local_os_info() {
  echo $(sudo cat /etc/os-release | grep "^NAME" | awk -F= '{print \$2}')
}

function get_centos_version() {
  local user=$1
  local host=$2
  echo $(ssh ${user}@${host} sudo cut -c22-24 /etc/redhat-release)
}

function get_ubuntu_cersion() {
  local user=$1
  local host=$2
  echo $(ssh ${user}@${host} sudo lsb_release -d | cut -c21-25)
}

function check_glusterfs_device() {
  local user=$1
  local node_ip=$2
  local node_device=$3

  local pass_or_ignore=false
  while ! ${pass_or_ignore}; do
    device_fmt=$(ssh ${user}@${node_ip} sudo parted -s $node_device print | grep 'Partition Table:')
    if [[ "$?" == "0" ]]; then
      if [[ "$device_fmt" != 'Partition Table: unknown' ]]; then
        echo "[$node_ip:$node_device]($device_fmt) 设备未初始化,请备份数据并对设备进行初始化,然后按Enter重试"
        #                echo "    初始化命令(数据会丢失): ssh $node_ip \"dd if=/dev/zero of=$node_device bs=512K count=1\""
        echo "    初始化命令(数据会丢失): ssh $node_ip \"pvcreate -y $node_device && pvremove -y $node_device \""
        read -p "或输入 reset 自动完成初始化(会导致设备数据丢失和机器重启,请谨慎操作!): " "reset"
        if [[ "$input_word" == "reset" ]]; then
          reset_cmd=$(ssh ${user}@$node_ip "sudo pvcreate -y $node_device && pvremove -y $node_device")
          if [[ "$?" == "0" ]]; then
            echo "设备初始化完成,执行机器重启操作"
            reboot_cmd=$(ssh ${user}@$node_ip reboot)
            echo "sleep 30s,等待机器重启完成"
            sleep 30
          else
            readp "设备初始化失败,请处理确认后按Enter继续: "
          fi
        fi
      else
        pass_or_ignore=true
      fi
    else
      readp "获取设备信息 [$node_ip:$node_device] 失败,请确认后按Enter重试,或输入 f 跳过该校验: " "f"
      if [[ $? -eq 0 ]]; then
        pass_or_ignore=true
      fi
    fi
  done
}

function wait_for_start() {
  local service_name=$1
  local sleep_seconds=$2
  echo "休眠 ${sleep_seconds}秒,等待 $service_name 启动完成"
  sleep ${sleep_seconds}
}

function init_database_local() {
  local database_host=$1
  local database=$2
  local sql_file=$3
  local utf8mb4=$4

  Info "初始化数据库: host=$database_host, database=$database, sql_file=$sql_file"

  if [[ -z "$database_host" ]]; then
    Warning "Master节点未配置,跳过数据库初始化"
    return 0
  fi

  mysql -uroot -pffffff -h${database_host} -e "show databases;" 2>&1
  if [[ $? -ne 0 ]]; then
    Error "执行mysql -uroot -pffffff -h${database_host} -e 'show databases' 失败"
    return 1
  fi

  local dbs=$(mysql -uroot -pffffff -h${database_host} -e 'show databases;' 2>&1)
  for db in ${dbs}; do
    if [[ "$db" == "$database" ]]; then
      table_rows=$(mysql -uroot -pffffff -h$database_host $database -e 'show tables;' 2>&1 | wc -l)
      if [[ ${table_rows} -gt 1 ]]; then
        if [[ "x${OVERWRITE_DATABASE}" == "xtrue" ]]; then
          Warning "检测到database [$database] 已存在,3秒后将会删除$database"
          backup_file_date_suffix=$(date +%s)
          Warning "备份数据中..."
          mysqldump -uroot -pffffff -h$database_host $database >/tmp/${database}_${backup_file_date_suffix}.sql 2>&1
          Warning "database [$database] 原有数据备份至: $master_host:/tmp/${database}_${backup_file_date_suffix}.sql "
          sleep 3
          Warning "删除database [$database]中..."
        else
          Warning "检测到database [$database] 已存在,跳过$sql_file 导入"
          return 0
        fi

      fi
    fi
  done
  mysql -uroot -pffffff -h$database_host -e "DROP DATABASE IF  EXISTS $database;" 2>&1
  if [[ "x${utf8mb4}" == "x1" ]]; then
    mysql -uroot -pffffff -h$database_host -e "CREATE DATABASE $database default character set utf8mb4 collate utf8mb4_unicode_ci;" 2>&1
  else
    mysql -uroot -pffffff -h$database_host -e "CREATE DATABASE IF NOT EXISTS $database;" 2>&1
  fi
  mysql -uroot -pffffff -h$database_host $database <$sql_file 2>&1
  if [[ $? -ne 0 ]]; then
    Error "导入$sql_file 失败"
    return 1
  fi
}

function update_database_local() {
  local database_host=$1
  local database=$2
  local sql_file=$3
  local utf8mb4=$4

  Info "初始化数据库: host=$database_host, database=$database, sql_file=$sql_file"

  if [[ -z "$database_host" ]]; then
    echo "Master节点未配置,跳过数据库初始化"
    return 0
  fi
  mysql -uroot -pffffff -h${database_host} -e "show databases;"
  if [[ $? -ne 0 ]]; then
    Error "执行mysql -uroot -pffffff -h${database_host} -e "show databases
    " 失败"
    return 1
  fi

  mysql -uroot -pffffff -h$database_host $database <$sql_file
  if [[ $? -ne 0 ]]; then
    Error "更新$sql_file 失败"
    return 1
  else
    Info "更新$sql_file 成功"
  fi
}

function init_database() {
  local user=$1
  local master_host=$2
  local database_host=$3
  local database=$4
  local sql_file=$5
  local utf8mb4=$6

  echo "初始化数据库: host=$database_host, database=$database, sql_file=$sql_file"

  if [[ -z "$master_host" ]]; then
    echo "Master节点未配置,跳过数据库初始化"
    return 0
  fi

  require_command_success_on_host ${user} ${master_host} "连接数据库" mysql -uroot -pffffff -h${database_host} -e "'show databases;'"

  local dbs=$(ssh ${user}@${master_host} mysql -uroot -pffffff -h${database_host} -e '"show databases;"')
  for db in ${dbs}; do
    if [[ "$db" == "$database" ]]; then
      table_rows=$(ssh ${user}@${master_host} "mysql -uroot -pffffff -h$database_host $database -e 'show tables;' | wc -l")
      if [[ ${table_rows} -gt 1 ]]; then
        readp "检测到database [$database] 已存在,是否删除原有数据,恢复成初始数据(y/n)? " "n"
        if [[ $? -eq 0 ]]; then
          return 1
        fi
        backup_file_date_suffix=$(ssh ${user}@${master_host} date +%s)
        ssh ${user}@${master_host} "mysqldump -uroot -pffffff -h$database_host $database > /tmp/${database}_${backup_file_date_suffix}.sql"
        echo "database [$database] 原有数据备份至: $master_host:/tmp/${database}_${backup_file_date_suffix}.sql "
      fi
    fi
  done
  ssh ${user}@${master_host} "mysql -uroot -pffffff -h$database_host -e 'DROP DATABASE IF  EXISTS $database;'"
  if [[ "x${utf8mb4}" == "x1" ]]; then
    ssh ${user}@${master_host} "mysql -uroot -pffffff -h$database_host -e 'CREATE DATABASE $database default character set utf8mb4 collate utf8mb4_unicode_ci;'"
  else
    ssh ${user}@${master_host} "mysql -uroot -pffffff -h$database_host -e 'CREATE DATABASE IF NOT EXISTS $database;'"
  fi
  ssh ${user}@${master_host} "mysql -uroot -pffffff -h$database_host $database < $sql_file"
}

function export_heketi_server_info() {
  local user=$1
  local master_host=$2

  export HEKETI_CLUSTER_IP="10.96.0.2"
  export HEKETI_CLI_SERVER="http://$HEKETI_CLUSTER_IP:8085"
  return 0

  local heketi_cluster_ip=$(ssh ${user}@${master_host} "kubectl -n $STEAMER_NAMESPACE describe svc/heketi | grep \"IP:\" | awk '{print \$2}'")
  if [[ $? -ne 0 || -z "$heketi_cluster_ip" ]]; then
    echo "获取heketi服务的Cluster IP失败,命令: ssh ${user}@$master_host \"kubectl -n $STEAMER_NAMESPACE describe svc/heketi | grep \\\"IP:\\\" | awk '{print \\\$2}'\""
    return 1
  fi
  export HEKETI_CLUSTER_IP=${heketi_cluster_ip}
  export HEKETI_CLI_SERVER="http://$heketi_cluster_ip:8085"
  ssh ${user}@${master_host} export HEKETI_CLUSTER_IP=${heketi_cluster_ip}
  ssh ${user}@${master_host} export HEKETI_CLI_SERVER="http://$heketi_cluster_ip:8085"
  return 0
}

function check_node_containers_status() {
  local user=$1
  local ip=$2
  local exited_container_count=$(ssh ${user}@${ip} "sudo docker ps -a | grep Exited | grep -v k8s_elasticsearch-logging-init | wc -l")
  if [ $exited_container_count -ne 0 ]; then
    ssh ${user}@${ip} "sudo docker ps -a | grep Exited"
    readp "节点 [${ip}] 存在异常退出的容器,是否自动删除容器让其自行重启(y/n)? " "y"
    if [[ $? -eq 0 ]]; then
      ssh ${user}@${ip} "sudo docker ps -a | grep Exited | awk '{print \$1}' | xargs -r sudo  docker rm"
      if [[ $? != 0 ]]; then
        echo "删除失败，请手动操作"
        return 1
      fi
    fi
  fi
  # 增加删除Dead容器
  local exited_container_count=$(ssh ${ip} "sudo docker ps -a | grep Dead | wc -l")
  if [[ ${exited_container_count} -ne 0 ]]; then
    ssh ${user}@${ip} "sudo docker ps -a | grep Dead"
    readp "节点 [${ip}] 存在异常退出的容器,是否自动删除容器让其自行重启(y/n)? " "y"
    if [[ $? -eq 0 ]]; then
      ssh ${user}@${ip} "sudo docker ps -a | grep Dead | awk '{print \$1}' | xargs -r sudo docker rm"
      if [[ $? != 0 ]]; then
        echo "删除失败，请手动操作"
        return 1
      fi
    fi
  fi
}

function check_node_containers_status_local() {
  local exited_container_count=$(docker ps -a | grep Exited | grep -v k8s_elasticsearch-logging-init | wc -l)
  if [ $exited_container_count -ne 0 ]; then
    ret=$(docker ps -a | grep Exited)
    Warning "节点存在异常退出的容器,自动删除异常容器中..."
    Warning "${ret}"
    docker ps -a | grep Exited | awk '{print $1}' | xargs -r sudo docker rm
    if [[ $? != 0 ]]; then
      Warning "删除失败，请手动处理"
      return 1
    else
      Warning "删除成功!"
    fi
  fi
  # 增加删除Dead容器
  local exited_container_count=$(sudo docker ps -a | grep Dead | wc -l)
  if [[ ${exited_container_count} -ne 0 ]]; then
    ret=$(docker ps -a | grep Dead)
    Warning "节点存在异常退出的容器,自动删除异常容器中..."
    Warning "${ret}"
    sudo docker ps -a | grep Dead | awk '{print $1}' | xargs -r sudo docker rm
    if [[ $? != 0 ]]; then
      Warning "删除失败，请手动处理"
      return 1
    else
      Warning "删除成功!"
    fi
  fi
}

function k8sBuild() {
  ns=$1
  file=$2
  msg=$(get_args_array 3 "$@")
  timeout=120

  which kubectl >/dev/null 2>&1
  if [[ $? -ne 0 ]]; then
    Error "$msg 找不到kubectl命令"
    exit 1
  fi
  kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
  if [[ $? -eq 0 ]]; then
    if [[ "x${RESOURCES_DEL}" == "xtrue" ]]; then
      kubectl delete -n ${ns} -f ${file}
      pass_or_ignore=false
      i=0

      for i in $(seq $timeout); do
        kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
        if [[ $? -ne 0 ]]; then
          Info "删除资源${file}成功!"
          break
        else
          Warning "正在删除资源${file}中..."
          sleep 1
        fi
      done
    fi
  fi

  kubectl apply -n ${ns} -f ${file}

  # 校验创建
  for i in $(seq $timeout); do
    kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
      Info "创建资源${file}成功"
      return 0
    else
      Warning "创建资源中..."
      sleep 1
    fi
    Error "创建资源${file}失败"
    exit 1
  done

}

function k8sApply() {
  ns=$1
  file=$2
  msg=$(get_args_array 3 "$@")
  timeout=120

  which kubectl >/dev/null 2>&1
  if [[ $? -ne 0 ]]; then
    Error "$msg 找不到kubectl命令"
    exit 1
  fi
  kubectl apply -n ${ns} -f ${file}

  # 校验创建
  for i in $(seq $timeout); do
    kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
      Info "创建资源${file}成功"
      return 0
    else
      Warning "创建资源中..."
      sleep 1
    fi
    Error "创建资源${file}失败"
    exit 1
  done

}

function k8sBuild1() {
  ns=$1
  file=$2
  msg=$(get_args_array 3 "$@")
  timeout=30

  which kubectl >/dev/null 2>&1
  if [[ $? -ne 0 ]]; then
    Error "$msg 找不到kubectl命令"
    exit 1
  fi
  kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
  if [[ $? -eq 0 ]]; then
    Warning "已经存在资源${file},跳过创建!"
    return 0
  fi

  kubectl apply -n ${ns} -f ${file}

  # 校验创建
  for i in $(seq $timeout); do
    kubectl get -n ${ns} -f ${file} >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
      Info "创建资源${file}成功"
      return 0
    else
      Warning "创建资源中..."
      sleep 1
    fi
    Error "创建资源${file}失败"
    exit 1
  done

}

function k8sDel() {
  ns=$1
  sel=$2
  kubectl delete pod -n $ns -l $sel
  if [[ $? -ne 0 ]]; then
    Error "删除资源[$sel]失败!"
  else
    Info "删除资源[$sel]成功!"
  fi
}

function rebuild_k8s_resources() {
  local user=$1
  local master_host=$2
  local namespace=$3
  local file_path_list=$(get_args_array 4 "$@")

  if [[ -z "$master_host" ]]; then
    echo "Master节点未配置,跳过Kubernetes资源创建"
    return 0
  fi

  for file_path in ${file_path_list}; do
    echo "创建服务的Kubernetes资源: namespace=$namespace, file=$file_path"

    ssh ${user}@${master_host} sudo kubectl get -n ${namespace} -f ${file_path} >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
      readp "服务的Kubernetes资源已存在,输入Enter自动删除并重建资源,或输入 f 跳过Kubernetes资源创建: " "f"
      if [[ $? -eq 0 ]]; then
        echo "跳过Kubernetes资源创建"
        return 0
      fi

      local pass_or_ignore=false
      while ! ${pass_or_ignore}; do
        echo "删除服务的历史Kubernetes资源"
        require_command_success_on_host ${user} ${master_host} "删除服务的历史Kubernetes资源"sudo kubectl delete -n ${namespace} -f $file_path
        sleep 8 # 等待8秒
        ssh ${user}@${master_host} sudo kubectl get -n ${namespace} -f $file_path >/dev/null 2>&1
        if [[ $? -eq 0 ]]; then
          readp "服务的Kubernetes资源删除失败,请确认后输入Enter重试,或输入 f 跳过该校验: " "f"
          if [[ $? -eq 0 ]]; then
            pass_or_ignore=true
          fi
        else
          pass_or_ignore=true
        fi
      done
    fi
    require_command_success_on_host ${user} ${master_host} "创建服务的Kubernetes资源"sudo kubectl apply -n $namespace -f $file_path
  done
}

function build_k8s_resources() {
  local user=$1
  local master_host=$2
  local namespace=$3
  local file_path_list=$(get_args_array 4 "$@")

  if [[ -z "$master_host" ]]; then
    echo "Master节点未配置,跳过Kubernetes资源创建"
    return 0
  fi

  for file_path in ${file_path_list}; do
    echo "创建服务的Kubernetes资源: namespace=$namespace, file=$file_path"

    ssh ${user}@${master_host} sudo kubectl get -n ${namespace} -f ${file_path} >/dev/null 2>&1
    if [[ $? -eq 0 ]]; then
      echo "服务的Kubernetes资源已存在,跳过Kubernetes资源创建"
    else
      require_command_success_on_host ${user} ${master_host} "创建服务的Kubernetes资源"sudo kubectl apply -n ${namespace} -f $file_path
    fi
  done
}

function push_image_to_registry() {
  local user=$1
  local master_host=$2
  local image_name=$3
  local image_version=$4

  # 本地执行
  if [ -z "$image_name" ]; then
    image_name=$1
    image_version=$2
    docker tag $image_name:$image_version hub.tiduyun.com:5000/$image_name:$image_version && docker push hub.tiduyun.com:5000/$image_name:$image_version
    if [[ $? -ne 0 ]]; then
      Error "推送镜像[$image_name]失败"
      exit 1
    fi
    return 0
  fi

  require_command_success_on_host ${user} ${master_host} "推送镜像 $image_name:$image_version 到镜像中心" "docker tag $image_name:$image_version hub.tiduyun.com:5000/$image_name:$image_version && docker push hub.tiduyun.com:5000/$image_name:$image_version"
}

function check_packets() {
  local user=$1
  local host=$2
  local name=$3

  #本地执行
  if [[ -z "$host" ]]; then
    local os_info=$(get_os_info)
    local name=$1
    if [[ ${os_info} =~ "CentOS" || ${os_info} =~ "Red Hat" ]]; then
      packets=$(rpm -qa | grep $name)
      if [[ ! -z ${packets} ]]; then
        Warning "发现安装包$packets  自动删除该软件包(yum remove $name*)"
        yum remove -y $name* 2>&1
      fi
    fi
    return
  fi

  # 远程执行
  local os_info=$(get_os_info ${user} ${host})

  if [[ ${os_info} =~ "CentOS" || ${os_info} =~ "Red Hat" ]]; then
    packets=$(ssh ${user}@${host} "sudo rpm -qa |grep $name")
    if [[ ! -z ${packets} ]]; then
      readp "在$host 机器上发现安装包$packets 输入 r 自动删除(yum remove $name*),或输入 f 跳过此步骤!" "f"
      if [[ $? -eq 0 ]]; then
        echo "跳过删除主机$host 上旧版本的$packets"
        return
      fi
      if [[ "$input_word" == "r" ]]; then
        ssh ${user}@${host} "sudo yum remove -y $name*"
      else
        echo "跳过删除主机$host 上旧版本的$packets"
      fi

    fi
  fi
}

function install_ubuntu_packer() {
  local user=$1
  local host=$2
  local path=$3
  local code=0
  # 本地执行
  if [[ -z ${host} ]]; then
    local path=$1
    if [[ ! -d ${path} ]]; then
      return 1
    fi
    for pk in $(sudo ls ${path}); do
      dpkg -i --force-all ${path}/${pk} 2>&1
      if [[ $? -ne 0 ]]; then
        #                Warning "执行dpkg -i ${path}/${pk}失败!"
        code=$(($code + 1))
      fi
    done
    # 该命令有时会卡住
    #        Warning "开始执行: apt install -f -y, 如该操作一直卡住不动,请手动执行该命令"
    #        apt install -f -y 2>&1
    #        if [[ $? -ne 0 ]]; then
    #            Warning "执行apt install -f -y失败!"
    #            code=$(($code + 1))
    #        fi
    #        Warning "执行: apt install -f -y 完成"
    Warning "开始执行: dpkg --configure -a --force-all, 如该操作一直卡住不动,请手动执行该命令"
    dpkg --configure -a --force-all 2>&1
    if [[ $? -ne 0 ]]; then
      Error "执行dpkg --config -a --force-all失败!"
      code=$(($code + 1))
    fi
    Warning "执行: dpkg --configure -a --force-all 完成"
    return $code
  fi

  #远程执行
  ssh -t ${user}@${host} "for pk in \$(sudo ls ${path}); do sudo dpkg -i ${path}/\${pk}; done"
  ssh_command ${user} ${host} "apt install -f -y"
  ssh_command ${user} ${host} "dpkg --configure -a --force-all"

}

function get_keepalive_peer_ip() {

  local the_ele="$1"
  local the_array=$(get_args_array 2 "$@")
  local index=0
  local ret=()
  for ele in ${the_array}; do
    if [[ "$ele" != "$the_ele" ]]; then
      ret[$index]="$ele"
      index=$(($index + 1))
    fi
  done
  echo ${ret[@]}
}

function md() {
  local path=$1
  if [[ ! -d ${path} ]]; then
    mkdir -p ${path}
    if [[ $? -ne 0 ]]; then
      Error "创建目录:  ${path} 失败"
      exit 1
    fi
  fi
}

# 以后优化的接口
function load() {
  file=$1
  if [[ ! -z $file ]]; then
    docker load -i ${file}
    if [[ $? -ne 0 ]]; then
      Error "导入镜像${file}失败"
      exit 1
    fi
  else
    for img in $(ls images); do
      docker load -i images/${img}
      if [[ $? -ne 0 ]]; then
        Error "导入镜像${file}失败"
        exit 1
      fi
    done
  fi
}

function loadPush() {
  file=$1
  Info "导入镜像${file}中..."
  context=$(docker load -i ${file})
  if [[ $? -ne 0 ]]; then
    Error "导入镜像[${file}]失败"
    exit 1
  fi
  name=$(echo $context | grep 'Loaded image:' | awk '{print $3}' | tr -d " " | tr -d "\t")
  if [[ -z ${name} ]]; then
    Error "未找到镜像名，请手动推送该镜像"
    exit 1
  fi
  echo "${name}" | grep "system" 2>&1
  if [[ $? -ne 0 ]]; then
    Warning "${name}未包含system仓库，跳过推送！"
    return 0
  fi

  echo "${name}" | grep "hub.tiduyun.com" 2>&1
  if [[ $? -ne 0 ]]; then
    docker tag ${name} hub.tiduyun.com:5000/${name}
  fi
  if [[ $? -ne 0 ]]; then
    Error "tag 镜像[${name}]失败"
    exit 1
  fi
  Info "推送镜像[${name}]中..."
  echo "${name}" | grep "hub.tiduyun.com" 2>&1
  if [[ $? -ne 0 ]]; then
    docker push hub.tiduyun.com:5000/${name}
  else
    docker push ${name}
  fi

  if [[ $? -ne 0 ]]; then
    Error "推送镜像[${name}]失败，请手动推送该镜像"
    exit 1
  fi
}

function checkErr() {
  code=$1
  msg=$2
  if [[ ${code} -ne 0 ]]; then
    Error "${msg} 执行失败!"
    exit 1
  fi
}

function get_nodes() {
  for i in $(seq 3); do
    num=$(kubectl get node --no-headers | wc -l)
    if [[ $? -eq 0 ]]; then
      break
    fi
  done
  echo $num
}

function getVer() {
  cmd="$@"
  ver=$($cmd >current_version.txt 2>&1)
  if [[ $? -ne 0 ]]; then
    Error "版本获取失败[$cmd]"
    return 1
  fi
  return 0
}

function appendVer() {
  cmd="$@"
  ver=$($cmd >>current_version.txt 2>&1)
  if [[ $? -ne 0 ]]; then
    Error "版本获取失败[$cmd]"
    return 1
  else
    Info "版本获取成功[$cmd]"
  fi
  return 0
}

function checkVer() {
  vMd5=$(cat version.txt | sort | md5sum | cut -d" " -f1)
  newMd5=$(cat current_version.txt | sort | md5sum | cut -d" " -f1)
  if [[ "x${vMd5}" != "x${newMd5}" ]]; then
    Error "版本校验失败"
    Error "安装包版本: $(cat version.txt)"
    Error "当前版本: $(cat current_version.txt)"
    return 1
  else
    Info "版本校验成功"
  fi
  return 0
}

function podName() {
  ns=$1
  sel=$2
  key=$3
  pod=$(kubectl get pod -n ${ns} -owide -l $sel 2>/dev/null | grep ${key} | awk '{print $1}' | head -n 1)
  if [[ -z $pod ]]; then
    Error "获取pod name失败 [kubectl get pod -n ${ns} -owide -l $sel | grep ${key} | awk '{print \$1}' | head -n 1]"
  fi
  echo "${pod}"
}

function resloveRepo() {
  mv /etc/yum.repos.d/steamer-bak/* /etc/yum.repos.d/ 2>&1
  rm -rf /etc/yum.repos.d/steamer-bak/ 2>&1
}

function fmtVersion() {
  ver=$1
  echo ${ver} | cut -c 1-3 | sed 's/\.//g'
}

function disk_iops_check() {
  local etcd_dir=$1
  local input_word=""
  local pass_or_ignore=false

  # local os_info=$(get_os_info)
  # if [[ $os_info =~ "Ubuntu" || $os_info =~ "Debian" ]]; then
  #     return 0
  # fi
  md ${etcd_dir}
  which fio 2>&1 >/dev/null
  if [[ $? == 0 ]]; then
    ret=$(sudo fio --rw=write --ioengine=sync --fdatasync=1 --directory=${etcd_dir} --size=22m --bs=2300 --name=mytest)
  else
    which docker 2>&1 >/dev/null
    if [[ $? == 0 ]]; then
      load steamer-tools.tar.gz
      ret=$(sudo docker run --rm -v ${etcd_dir}:${etcd_dir} steamer-tools:v1.0 fio --rw=write --ioengine=sync --fdatasync=1 --directory=${etcd_dir} --size=22m --bs=2300 --name=mytest)
    else
      Warning "未进行iops测试"
      return
    fi
  fi
  iops=$(echo "${ret}" | grep iops | awk '{print $4}' | sed 's/,//g')
  if [[ -z ${iops} ]]; then
    iops=0
  fi
  fsync=$(echo "${ret}" | grep "99.00th\|fsync" | tail -n 1 | awk -F[\]\[\] '{print $2}' | sed 's/\]//g' | sed 's/,//g' | sed 's/ //g')
  local iops_ok=false
  local fsync_ok=false

  if [[ ${iops} -lt 50 ]]; then
    Error "min_iops=${iops},小于50(etcd最低要求),会影响etcd的数据存储(该条件必须满足)"
    code=$(($code + 1))
    Failed=$(($Failed + 1))
    exit 100
  fi

  if [[ ${fsync} -gt 10000 ]]; then
    Error "fdatasync 99th percentile =${fsync}usec ,大于10ms(etcd最低要求),会影etcd的数据存储"
    code=$(($code + 1))
    exit 100
  fi
  Info "iops: [$iops], fsync: [${fsync}]"
}

function isAllInOne() {
  local masterCount=${#STEAMER_MASTER_HOSTS[@]}
  local nodeCount=${#STEAMER_AGENT_HOSTS[@]}

  if [[ ${masterCount} -eq 1 ]] && [[ ${nodeCount} -eq 1 ]] && [[ ${STEAMER_MASTER_HOSTS[0]} == ${STEAMER_AGENT_HOSTS[0]} ]]; then
    echo true
  else
    echo false
  fi

}

function isHa() {
  local masterCount=${#STEAMER_MASTER_HOSTS[@]}
  local nodeCount=${#STEAMER_AGENT_HOSTS[@]}
  if [[ ${masterCount} -eq 3 ]]; then
    echo true
  else
    echo false
  fi

}

function isSingleton() {
  local masterCount=${#STEAMER_MASTER_HOSTS[@]}
  local nodeCount=${#STEAMER_AGENT_HOSTS[@]}
  if [[ ${masterCount} -eq 1 ]]; then
    if [[ ${nodeCount} -eq 1 ]] && [[ ${STEAMER_MASTER_HOSTS[0]} == ${STEAMER_AGENT_HOSTS[0]} ]]; then
      echo false
    else
      echo true
    fi
  else
    echo false
  fi
}

function isMaster() {
  local m1=${MASTER1}
  local m2=${MASTER2}
  local m3=${MASTER3}
  local cip=${CURRENT_IP}
  if [[ ! -z ${m1} ]] && [[ "x${cip}" == "x${m1}" ]]; then
    echo "true"
    return 0
  fi

  if [[ ! -z ${m2} ]] && [[ "x${cip}" == "x${m2}" ]]; then
    echo "true"
    return 0
  fi

  if [[ ! -z ${m3} ]] && [[ "x${cip}" == "x${m3}" ]]; then
    echo "true"
    return 0
  fi

  echo "false"
  return 1
}

function isMaster1() {
  local m1=${MASTER1}
  local cip=${CURRENT_IP}
  if [[ ! -z ${m1} ]] && [[ "x${cip}" == "x${m1}" ]]; then
    echo "true"
    return 0
  fi
  echo "false"
  return 1
}

# 当调用PUSH/UPDATE变量为true时，在master1　push镜像，其他节点load镜像,否則只load鏡像

function pushImages() {
  if [[ -z ${PUSH} ]]; then
    PUSH=false
  fi
  if [[ -z ${UPDATE} ]]; then
    UPDATE=false
  fi
  for img in $(ls images); do
    if ${PUSH} || ${UPDATE}; then
      if $(isMaster1); then
        loadPush images/$img
      else
        load images/$img
      fi
    else
      load images/$img
    fi
  done
}