deploy.sh 34.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
#!/bin/bash

# Deploy script to install provisioning server for OPNFV Apex
# author: Dan Radez (dradez@redhat.com)
# author: Tim Rozet (trozet@redhat.com)
#
# Based on RDO Manager http://www.rdoproject.org

set -e

##VARIABLES
12 13 14 15 16 17 18 19 20 21 22
if [ "$TERM" != "unknown" ]; then
  reset=$(tput sgr0)
  blue=$(tput setaf 4)
  red=$(tput setaf 1)
  green=$(tput setaf 2)
else
  reset=""
  blue=""
  red=""
  green=""
fi
23

Tim Rozet's avatar
Tim Rozet committed
24
vm_index=4
Dan Radez's avatar
Dan Radez committed
25
ha_enabled="TRUE"
Dan Radez's avatar
Dan Radez committed
26 27
ping_site="8.8.8.8"
ntp_server="pool.ntp.org"
28 29
net_isolation_enabled="TRUE"

30 31
declare -i CNT
declare UNDERCLOUD
32
declare -A deploy_options_array
33
declare -A NET_MAP
34

35
SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
36 37 38
DEPLOY_OPTIONS=""
RESOURCES=/var/opt/opnfv/stack
CONFIG=/var/opt/opnfv
39
INSTACKENV=$CONFIG/instackenv.json
40 41 42 43 44 45 46 47 48
OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
# Netmap used to map networks to OVS bridge names
NET_MAP['admin_network']="brbm"
NET_MAP['private_network']="brbm1"
NET_MAP['public_network']="brbm2"
NET_MAP['storage_network']="brbm3"

##LIBRARIES
source $CONFIG/lib/common-functions.sh
49

50
##FUNCTIONS
51 52 53 54 55 56 57 58 59 60 61 62 63 64
##translates yaml into variables
##params: filename, prefix (ex. "config_")
##usage: parse_yaml opnfv_ksgen_settings.yml "config_"
parse_yaml() {
   local prefix=$2
   local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
   sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  $1 |
   awk -F$fs '{
      indent = length($1)/2;
      vname[indent] = $2;
      for (i in vname) {if (i > indent) {delete vname[i]}}
      if (length($3) > 0) {
         vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
65
         printf("%s%s%s=%s\n", "'$prefix'",vn, $2, $3);
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
      }
   }'
}

##checks if prefix exists in string
##params: string, prefix
##usage: contains_prefix "deploy_setting_launcher=1" "deploy_setting"
contains_prefix() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^$prefix.*$" > /dev/null; then
    return 0
  else
    return 1
  fi
}
##parses variable from a string with '='
##and removes global prefix
##params: string, prefix
##usage: parse_setting_var 'deploy_myvar=2' 'deploy_'
parse_setting_var() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^.+\=" > /dev/null; then
    echo $(echo $mystr | grep -Eo "^.+\=" | tr -d '=' |  sed 's/^'"$prefix"'//')
  else
    return 1
  fi
}
##parses value from a string with '='
##params: string
##usage: parse_setting_value
parse_setting_value() {
  local mystr=$1
  echo $(echo $mystr | grep -Eo "\=.*$" | tr -d '=')
}
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
##parses network settings yaml into globals
parse_network_settings() {
  local required_network_settings="cidr"
  local common_optional_network_settings="usable_ip_range"
  local admin_network_optional_settings="provisioner_ip dhcp_range introspection_range"
  local public_network_optional_settings="floating_ip_range gateway provisioner_ip"
  local nic_value cidr

  eval $(parse_yaml ${NETSETS})
  for network in ${OPNFV_NETWORK_TYPES}; do
    if [[ $(eval echo \${${network}_enabled}) == 'true' ]]; then
      enabled_network_list+="${network} "
    elif [ "${network}" == 'admin_network' ]; then
      echo -e "${red}ERROR: You must enable admin_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    elif [ "${network}" == 'public_network' ]; then
      echo -e "${red}ERROR: You must enable public_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    else
      echo -e "${blue}INFO: Network: ${network} is disabled, will collapse into admin_network"
    fi
  done

  # check for enabled network values
  for enabled_network in ${enabled_network_list}; do
    # detect required settings first to continue
    echo -e "${blue}INFO: Detecting Required settings for: ${enabled_network}${reset}"
    for setting in ${required_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        # if setting is missing we try to autodetect
        eval "nic_value=\${${enabled_network}_bridged_interface}"
        if [ -n "$nic_value" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-detection failed: ${setting} not found using interface: ${nic_value}${reset}"
            exit 1
          fi
        else
          echo -e "${red}ERROR: Required setting: ${setting} not found, and bridge interface not provided\
for Auto-detection${reset}"
          exit 1
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Common settings for: ${enabled_network}${reset}"
    # detect optional common settings
    # these settings can be auto-generated if missing
    for setting in ${common_optional_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        setting_value=$(eval find_${setting} ${nic_value})
        if [ -n "$setting_value" ]; then
          eval "${enabled_network}_${setting}=${setting_value}"
          echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
        else
          # if Auto-detection fails we can auto-generate with CIDR
          eval "cidr=\${${enabled_network}_cidr}"
          setting_value=$(eval generate_${setting} ${cidr})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
            exit 1
          fi
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Network Specific settings for: ${enabled_network}${reset}"
    # detect network specific settings
    if [ -n $(eval echo \${${network}_optional_settings}) ]; then
      eval "network_specific_settings=\${${enabled_network}_optional_settings}"
      for setting in ${network_specific_settings}; do
        eval "setting_value=\${${enabled_network}_${setting}}"
        if [ -z "${setting_value}" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            eval "cidr=\${${enabled_network}_cidr}"
            setting_value=$(eval generate_${setting} ${cidr})
            if [ -n "$setting_value" ]; then
              eval "${enabled_network}_${setting}=${setting_value}"
              echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
            else
              echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
              exit 1
            fi
          fi
        else
          echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
        fi
      done
    fi
  done
}
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
##parses deploy settings yaml into globals and options array
##params: none
##usage:  parse_deploy_settings
parse_deploy_settings() {
  local global_prefix="deploy_global_params_"
  local options_prefix="deploy_deploy_options_"
  local myvar myvalue
  local settings=$(parse_yaml $DEPLOY_SETTINGS_FILE "deploy_")

  for this_setting in $settings; do
    if contains_prefix $this_setting $global_prefix; then
      myvar=$(parse_setting_var $this_setting $global_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      # Do not override variables set by cmdline
      if [ -z "$(eval echo \$$myvar)" ]; then
        eval "$myvar=\$myvalue"
        echo -e "${blue}Global parameter set: ${myvar}:${myvalue}${reset}"
      else
        echo -e "${blue}Global parameter already set: ${myvar}${reset}"
      fi
    elif contains_prefix $this_setting $options_prefix; then
      myvar=$(parse_setting_var $this_setting $options_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      deploy_options_array[$myvar]=$myvalue
      echo -e "${blue}Deploy option set: ${myvar}:${myvalue}${reset}"
    fi
  done
}
##parses baremetal yaml settings into compatible json
##writes the json to $CONFIG/instackenv_tmp.json
##params: none
##usage: parse_inventory_file
parse_inventory_file() {
  local inventory=$(parse_yaml $INVENTORY_FILE)
  local node_list
  local node_prefix="node"
  local node_count=0
  local node_total
  local inventory_list

  # detect number of nodes
  for entry in $inventory; do
    if echo $entry | grep -Eo "^nodes_node[0-9]+_" > /dev/null; then
      this_node=$(echo $entry | grep -Eo "^nodes_node[0-9]+_")
      if [[ $inventory_list != *"$this_node"* ]]; then
        inventory_list+="$this_node "
      fi
    fi
  done

  inventory_list=$(echo $inventory_list | sed 's/ $//')

  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
266
    ((node_count+=1))
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
  done

  node_total=$node_count

  if [[ "$node_total" -lt 5 && ha_enabled == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide at least 5 nodes for HA baremetal deployment${reset}"
    exit 1
  elif [[ "$node_total" -lt 2 ]]; then
    echo -e "${red}ERROR: You must provide at least 2 nodes for non-HA baremetal deployment${reset}"
    exit 1
  fi

  eval $(parse_yaml $INVENTORY_FILE)

  instack_env_output="
{
 \"nodes\" : [

"
  node_count=0
  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
288 289
    ((node_count+=1))
    node_output="
290 291 292 293 294 295 296 297 298 299 300
        {
          \"pm_password\": \"$(eval echo \${${node}ipmi_pass})\",
          \"pm_type\": \"pxe_ipmitool\",
          \"mac\": [
            \"$(eval echo \${${node}mac_address})\"
          ],
          \"cpu\": \"$(eval echo \${${node}cpus})\",
          \"memory\": \"$(eval echo \${${node}memory})\",
          \"disk\": \"$(eval echo \${${node}disk})\",
          \"arch\": \"$(eval echo \${${node}arch})\",
          \"pm_user\": \"$(eval echo \${${node}ipmi_user})\",
301 302
          \"pm_addr\": \"$(eval echo \${${node}ipmi_ip})\",
          \"capabilities\": \"$(eval echo \${${node}capabilities})\"
303
"
Dan Radez's avatar
Dan Radez committed
304
    instack_env_output+=${node_output}
305 306 307 308 309 310 311 312 313 314 315 316 317
    if [ $node_count -lt $node_total ]; then
      instack_env_output+="        },"
    else
      instack_env_output+="        }"
    fi
  done

  instack_env_output+='
  ]
}
'
  #Copy instackenv.json to undercloud for baremetal
  echo -e "{blue}Parsed instackenv JSON:\n${instack_env_output}${reset}"
Dan Radez's avatar
Dan Radez committed
318
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
319 320 321 322 323 324
cat > instackenv.json << EOF
$instack_env_output
EOF
EOI

}
325 326 327
##verify internet connectivity
#params: none
function verify_internet {
Dan Radez's avatar
Dan Radez committed
328
  if ping -c 2 $ping_site > /dev/null; then
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
    if ping -c 2 www.google.com > /dev/null; then
      echo "${blue}Internet connectivity detected${reset}"
      return 0
    else
      echo "${red}Internet connectivity detected, but DNS lookup failed${reset}"
      return 1
    fi
  else
    echo "${red}No internet connectivity detected${reset}"
    return 1
  fi
}

##download dependencies if missing and configure host
#params: none
function configure_deps {
  if ! verify_internet; then
    echo "${red}Will not download dependencies${reset}"
    internet=false
Dan Radez's avatar
Dan Radez committed
348 349 350 351 352 353
  fi

  # verify ip forwarding
  if sysctl net.ipv4.ip_forward | grep 0; then
    sudo sysctl -w net.ipv4.ip_forward=1
    sudo sh -c "echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf"
354 355
  fi

356 357 358 359 360 361 362 363
  # ensure no dhcp server is running on jumphost
  if ! sudo systemctl status dhcpd | grep dead; then
    echo "${red}WARN: DHCP Server detected on jumphost, disabling...${reset}"
    sudo systemctl stop dhcpd
    sudo systemctl disable dhcpd
  fi

  # ensure networks are configured
364
  systemctl start openvswitch
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399

  # If flat we only use admin network
  if [[ "$net_isolation_enabled" == "FALSE" ]]; then
    virsh_enabled_networks="admin_network"
  # For baremetal we only need to create/attach instack to admin and public
  elif [ "$virtual" == "FALSE" ]; then
    virsh_enabled_networks="admin_network public_network"
  else
    virsh_enabled_neworks=$enabled_network_list
  fi

  for network in ${OPNFV_NETWORK_TYPES}; do
    ovs-vsctl list-br | grep ${NET_MAP[$network]} > /dev/null || ovs-vsctl add-br ${NET_MAP[$network]}
    virsh net-list --all | grep ${NET_MAP[$network]} > /dev/null || virsh net-create $CONFIG/${NET_MAP[$network]}-net.xml
    virsh net-list | grep -E "${NET_MAP[$network]}\s+active" > /dev/null || virsh net-start ${NET_MAP[$network]}
  done

  echo -e "${blue}INFO: Bridges set: ${reset}"
  ovs-vsctl list-br
  echo -e "${blue}INFO: virsh networks set: ${reset}"
  virsh net-list

  if [[ -z "$virtual" || "$virtual" == "FALSE" ]]; then
    # bridge interfaces to correct OVS instances for baremetal deployment
    for network in ${enabled_network_list}; do
      this_interface=$(eval echo \${${network}_bridged_interface})
      # check if this a bridged interface for this network
      if [[ -n "$this_interface" || "$this_interface" != "none" ]]; then
        ovs-vsctl list-ports ${NET_MAP[$network]} | grep ${this_interface} || ovs-vsctl add-port ${NET_MAP[$network]} ${this_interface}
      else
        echo "${red}ERROR: Unable to determine interface to bridge to for enabled network: ${network}${reset}"
        exit 1
      fi
    done
  fi
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422

  # ensure storage pool exists and is started
  virsh pool-list --all | grep default > /dev/null || virsh pool-create $CONFIG/default-pool.xml
  virsh pool-list | grep -Eo "default\s+active" > /dev/null || virsh pool-start default

  if virsh net-list | grep default > /dev/null; then
    num_ints_same_subnet=$(ip addr show | grep "inet 192.168.122" | wc -l)
    if [ "$num_ints_same_subnet" -gt 1 ]; then
      virsh net-destroy default
      ##go edit /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' instackenv-virt.json
      sleep 5
      virsh net-start default
      virsh net-autostart default
    fi
  fi

  if ! egrep '^flags.*(vmx|svm)' /proc/cpuinfo > /dev/null; then
    echo "${red}virtualization extensions not found, kvm kernel module insertion may fail.\n  \
Are you sure you have enabled vmx in your bios or hypervisor?${reset}"
  fi

Dan Radez's avatar
Dan Radez committed
423 424
  if ! lsmod | grep kvm > /dev/null; then modprobe kvm; fi
  if ! lsmod | grep kvm_intel > /dev/null; then modprobe kvm_intel; fi
425 426 427 428 429 430 431 432 433 434 435 436 437

  if ! lsmod | grep kvm > /dev/null; then
    echo "${red}kvm kernel modules not loaded!${reset}"
    return 1
  fi

  ##sshkeygen for root
  if [ ! -e ~/.ssh/id_rsa.pub ]; then
    ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
  fi

  echo "${blue}All dependencies installed and running${reset}"
}
438 439

##verify vm exists, an has a dhcp lease assigned to it
Dan Radez's avatar
Dan Radez committed
440
##params: none
441
function setup_instack_vm {
442
  if ! virsh list --all | grep instack > /dev/null; then
443
      #virsh vol-create default instack.qcow2.xml
444
      virsh define $CONFIG/instack.xml
445

446 447 448 449
      #Upload instack image
      #virsh vol-create default --file instack.qcow2.xml
      virsh vol-create-as default instack.qcow2 30G --format qcow2

450 451 452 453 454 455 456 457 458
      ### this doesn't work for some reason I was getting hangup events so using cp instead
      #virsh vol-upload --pool default --vol instack.qcow2 --file $CONFIG/stack/instack.qcow2
      #2015-12-05 12:57:20.569+0000: 8755: info : libvirt version: 1.2.8, package: 16.el7_1.5 (CentOS BuildSystem <http://bugs.centos.org>, 2015-11-03-13:56:46, worker1.bsys.centos.org)
      #2015-12-05 12:57:20.569+0000: 8755: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #2015-12-05 12:57:20.569+0000: 8756: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #error: cannot close volume instack.qcow2
      #error: internal error: received hangup / error event on socket
      #error: Reconnected to the hypervisor

459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
      instack_dst=/var/lib/libvirt/images/instack.qcow2
      cp -f $RESOURCES/instack.qcow2 $instack_dst

      # resize instack machine
      echo "Checking if instack needs to be resized..."
      instack_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep device | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
      if [ "$instack_size" -lt 30 ]; then
        qemu-img resize /var/lib/libvirt/images/instack.qcow2 +25G
	LIBGUESTFS_BACKEND=direct virt-resize --expand /dev/sda1 $RESOURCES/instack.qcow2 $instack_dst
	LIBGUESTFS_BACKEND=direct virt-customize -a $instack_dst --run-command 'xfs_growfs -d /dev/sda1 || true'
        new_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep filesystem | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
        if [ "$new_size" -lt 30 ]; then
          echo "Error resizing instack machine, disk size is ${new_size}"
          exit 1
        else
          echo "instack successfully resized"
        fi
      else
        echo "skipped instack resize, upstream is large enough"
      fi
479 480 481 482

  else
      echo "Found Instack VM, using existing VM"
  fi
483 484 485 486

  # if the VM is not running update the authkeys and start it
  if ! virsh list | grep instack > /dev/null; then
    echo "Injecting ssh key to instack VM"
487 488
    virt-customize -c qemu:///system -d instack --run-command "mkdir /root/.ssh/" \
        --upload ~/.ssh/id_rsa.pub:/root/.ssh/authorized_keys \
489 490 491 492 493 494
        --run-command "chmod 600 /root/.ssh/authorized_keys && restorecon /root/.ssh/authorized_keys" \
        --run-command "cp /root/.ssh/authorized_keys /home/stack/.ssh/" \
        --run-command "chown stack:stack /home/stack/.ssh/authorized_keys && chmod 600 /home/stack/.ssh/authorized_keys"
    virsh start instack
  fi

495 496 497
  sleep 3 # let DHCP happen

  CNT=10
498 499
  echo -n "${blue}Waiting for instack's dhcp address${reset}"
  while ! grep instack /var/lib/libvirt/dnsmasq/default.leases > /dev/null && [ $CNT -gt 0 ]; do
500 501 502 503 504 505
      echo -n "."
      sleep 3
      CNT=CNT-1
  done

  # get the instack VM IP
506
  UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
Dan Radez's avatar
Dan Radez committed
507
  if [ -z "$UNDERCLOUD" ]; then
508 509 510
    #if not found then dnsmasq may be using leasefile-ro
    instack_mac=$(virsh domiflist instack | grep default | \
                  grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
511
    UNDERCLOUD=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
512 513 514 515

    if [ -z "$UNDERCLOUD" ]; then
      echo "\n\nNever got IP for Instack. Can Not Continue."
      exit 1
516 517
    else
      echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
518
    fi
Dan Radez's avatar
Dan Radez committed
519 520 521
  else
     echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
  fi
522 523

  CNT=10
524
  echo -en "${blue}\rValidating instack VM connectivity${reset}"
525 526 527
  while ! ping -c 1 $UNDERCLOUD > /dev/null && [ $CNT -gt 0 ]; do
      echo -n "."
      sleep 3
528
      CNT=$CNT-1
529
  done
Dan Radez's avatar
Dan Radez committed
530
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
531 532 533
      echo "Failed to contact Instack. Can Not Continue"
      exit 1
  fi
534
  CNT=10
535
  while ! ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "echo ''" 2>&1> /dev/null && [ $CNT -gt 0 ]; do
536 537
      echo -n "."
      sleep 3
538
      CNT=$CNT-1
539
  done
Dan Radez's avatar
Dan Radez committed
540
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
541 542 543
      echo "Failed to connect to Instack. Can Not Continue"
      exit 1
  fi
544 545

  # extra space to overwrite the previous connectivity output
Dan Radez's avatar
Dan Radez committed
546
  echo -e "${blue}\r                                                                 ${reset}"
547

548 549 550 551 552 553
  #add the instack public interface if net isolation is enabled (more than just admin network)
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    virsh attach-interface --domain instack --type network --source ${NET_MAP['public_network']} --model rtl8139 --config --live
    sleep 1
    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "if ! ip a s eth2 | grep ${public_network_provisioner_ip} > /dev/null; then ip a a ${public_network_provisioner_ip}/${public_network_cidr##*/} dev eth2; ip link set up dev eth2; fi"
  fi
554 555 556
  # ssh key fix for stack user
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "restorecon -r /home/stack"
}
557

Tim Rozet's avatar
Tim Rozet committed
558 559
##Create virtual nodes in virsh
##params: none
560 561
function setup_virtual_baremetal {
  for i in $(seq 0 $vm_index); do
562 563 564
    if ! virsh list --all | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} > /dev/null; then
      if [ ! -e $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml ]; then
        define_virtual_node baremetalbrbm_brbm1_brbm2_brbm3_${i}
Tim Rozet's avatar
Tim Rozet committed
565
      fi
566 567 568 569 570 571 572
      # Fix for ramdisk using wrong pxeboot interface
      # TODO: revisit this and see if there's a more proper fix
      sed -i "/^\s*<source network='brbm2'\/>/{
        N
        s/^\(.*\)virtio\(.*\)$/\1rtl8139\2/
        }" $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
      virsh define $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
573 574 575
    else
      echo "Found Baremetal ${i} VM, using existing VM"
    fi
576
    virsh vol-list default | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} 2>&1> /dev/null || virsh vol-create-as default baremetalbrbm_brbm1_brbm2_brbm3_${i}.qcow2 40G --format qcow2
577
  done
578

579 580
}

581 582 583 584 585 586 587 588 589
##Set network-environment settings
##params: network-environment file to edit
function configure_network_environment {
  sed -i '/ControlPlaneSubnetCidr/c\\  ControlPlaneSubnetCidr: "'${admin_network_cidr##*/}'"' $1
  sed -i '/ControlPlaneDefaultRoute/c\\  ControlPlaneDefaultRoute: '${admin_network_provisioner_ip}'' $1
  sed -i '/ExternalNetCidr/c\\  ExternalNetCidr: '${public_network_cidr}'' $1
  sed -i "/ExternalAllocationPools/c\\  ExternalAllocationPools: [{'start': '${public_network_usable_ip_range%%,*}', 'end': '${public_network_usable_ip_range##*,}'}]" $1
  sed -i '/ExternalInterfaceDefaultRoute/c\\  ExternalInterfaceDefaultRoute: '${public_network_gateway}'' $1
}
590
##Copy over the glance images and instack json file
Tim Rozet's avatar
Tim Rozet committed
591
##params: none
592
function configure_undercloud {
593

594 595
  echo
  echo "Copying configuration file and disk images to instack"
596
  scp ${SSH_OPTIONS[@]} $RESOURCES/overcloud-full.qcow2 "stack@$UNDERCLOUD":
597 598 599 600 601 602
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    configure_network_environment $CONFIG/network-environment.yaml
    echo -e "${blue}Network Environment set for Deployment: ${reset}"
    cat $CONFIG/network-environment.yaml
    scp ${SSH_OPTIONS[@]} $CONFIG/network-environment.yaml "stack@$UNDERCLOUD":
  fi
Dan Radez's avatar
Dan Radez committed
603
  scp ${SSH_OPTIONS[@]} -r $CONFIG/nics/ "stack@$UNDERCLOUD":
604

605 606 607
  # ensure stack user on instack machine has an ssh key
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "if [ ! -e ~/.ssh/id_rsa.pub ]; then ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa; fi"

608
  if [ "$virtual" == "TRUE" ]; then
609 610 611 612 613 614

      # copy the instack vm's stack user's pub key to
      # root's auth keys so that instack can control
      # vm power on the hypervisor
      ssh ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys

615 616 617 618 619 620 621
      # fix MACs to match new setup
      for i in $(seq 0 $vm_index); do
        pyscript="import json
data = json.load(open('$CONFIG/instackenv-virt.json'))
print data['nodes'][$i]['mac'][0]"

        old_mac=$(python -c "$pyscript")
622
        new_mac=$(virsh dumpxml baremetalbrbm_brbm1_brbm2_brbm3_$i | grep "mac address" | cut -d = -f2 | grep -Eo "[0-9a-f:]+")
623
        # this doesn't work with multiple vnics on the vms
Dan Radez's avatar
Dan Radez committed
624 625 626 627
        #if [ "$old_mac" != "$new_mac" ]; then
        #  echo "${blue}Modifying MAC for node from $old_mac to ${new_mac}${reset}"
        #  sed -i 's/'"$old_mac"'/'"$new_mac"'/' $CONFIG/instackenv-virt.json
        #fi
628 629
      done

630
      DEPLOY_OPTIONS+=" --libvirt-type qemu"
631
      INSTACKENV=$CONFIG/instackenv-virt.json
632

633 634 635
      # upload instackenv file to Instack for virtual deployment
      scp ${SSH_OPTIONS[@]} $INSTACKENV "stack@$UNDERCLOUD":instackenv.json
  fi
636 637 638

  # allow stack to control power management on the hypervisor via sshkey
  # only if this is a virtual deployment
639
  if [ "$virtual" == "TRUE" ]; then
640 641 642 643 644 645 646 647 648
      ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
while read -r line; do
  stack_key=\${stack_key}\\\\\\\\n\${line}
done < <(cat ~/.ssh/id_rsa)
stack_key=\$(echo \$stack_key | sed 's/\\\\\\\\n//')
sed -i 's~INSERT_STACK_USER_PRIV_KEY~'"\$stack_key"'~' instackenv.json
EOI
  fi

649 650 651
  # copy stack's ssh key to this users authorized keys
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> ~/.ssh/authorized_keys

652 653 654
  # disable requiretty for sudo
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "sed -i 's/Defaults\s*requiretty//'" /etc/sudoers

655 656 657 658
  # configure undercloud on Undercloud VM
  echo "Running undercloud configuration."
  echo "Logging undercloud configuration to instack:/home/stack/apex-undercloud-install.log"
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" << EOI
659
if [[ "$net_isolation_enabled" == "TRUE" ]]; then
660 661 662 663 664 665 666 667
  sed -i 's/#local_ip/local_ip/' undercloud.conf
  sed -i 's/#network_gateway/network_gateway/' undercloud.conf
  sed -i 's/#network_cidr/network_cidr/' undercloud.conf
  sed -i 's/#dhcp_start/dhcp_start/' undercloud.conf
  sed -i 's/#dhcp_end/dhcp_end/' undercloud.conf
  sed -i 's/#inspection_iprange/inspection_iprange/' undercloud.conf
  sed -i 's/#undercloud_debug/undercloud_debug/' undercloud.conf

668 669 670 671 672 673
  openstack-config --set undercloud.conf DEFAULT local_ip ${admin_network_provisioner_ip}/${admin_network_cidr##*/}
  openstack-config --set undercloud.conf DEFAULT network_gateway ${admin_network_provisioner_ip}
  openstack-config --set undercloud.conf DEFAULT network_cidr ${admin_network_cidr}
  openstack-config --set undercloud.conf DEFAULT dhcp_start ${admin_network_dhcp_range%%,*}
  openstack-config --set undercloud.conf DEFAULT dhcp_end ${admin_network_dhcp_range##*,}
  openstack-config --set undercloud.conf DEFAULT inspection_iprange ${admin_network_introspection_range}
674 675 676 677
  openstack-config --set undercloud.conf DEFAULT undercloud_debug false
fi

openstack undercloud install &> apex-undercloud-install.log
678 679 680 681
sleep 30
sudo systemctl restart openstack-glance-api
sudo systemctl restart openstack-nova-conductor
sudo systemctl restart openstack-nova-compute
682
EOI
683 684 685 686 687 688 689
# WORKAROUND: must restart the above services to fix sync problem with nova compute manager
# TODO: revisit and file a bug if necessary. This should eventually be removed
# as well as glance api problem
echo -e "${blue}INFO: Sleeping 15 seconds while services come back from restart${reset}"
sleep 15
#TODO Fill in the rest of the network-environment values for other networks

690 691 692
}

##preping it for deployment and launch the deploy
Tim Rozet's avatar
Tim Rozet committed
693
##params: none
694
function undercloud_prep_overcloud_deploy {
695 696 697 698 699 700

  if [[ ${#deploy_options_array[@]} -eq 0 || ${deploy_options_array['sdn_controller']} == 'opendaylight' ]]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'opendaylight-external' ]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight-external.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'onos' ]; then
bobzh's avatar
bobzh committed
701
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/onos.yaml"
702 703 704 705
  elif [ ${deploy_options_array['sdn_controller']} == 'opencontrail' ]; then
    echo -e "${red}ERROR: OpenContrail is currently unsupported...exiting${reset}"
    exit 1
  fi
706

Tim Rozet's avatar
Tim Rozet committed
707
  # check if HA is enabled
708
  if [[ "$ha_enabled" == "TRUE" ]]; then
Tim Rozet's avatar
Tim Rozet committed
709 710 711
     DEPLOY_OPTIONS+=" --control-scale 3 --compute-scale 2"
     DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/puppet-pacemaker.yaml"
  fi
712

713
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
714
     #DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml"
715
     DEPLOY_OPTIONS+=" -e network-environment.yaml"
716 717
  fi

718
  if [[ "$ha_enabled" == "TRUE" ]] || [[ "$net_isolation_enabled" == "TRUE" ]]; then
Dan Radez's avatar
Dan Radez committed
719
     DEPLOY_OPTIONS+=" --ntp-server $ntp_server"
720 721
  fi

722 723 724 725
  if [[ ! "$virtual" == "TRUE" ]]; then
     DEPLOY_OPTIONS+=" --control-flavor control --compute-flavor compute"
  fi

726
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
727
source stackrc
728 729
set -o errexit
echo "Uploading overcloud glance images"
730
openstack overcloud image upload
731
echo "Configuring undercloud and discovering nodes"
732 733 734
openstack baremetal import --json instackenv.json
openstack baremetal configure boot
openstack baremetal introspection bulk start
735
echo "Configuring flavors"
736 737 738 739 740 741 742 743 744 745
for flavor in baremetal control compute; do
  echo -e "${blue}INFO: Updating flavor: \${flavor}${reset}"
  if openstack flavor list | grep \${flavor}; then
    openstack flavor delete \${flavor}
  fi
  openstack flavor create --id auto --ram 4096 --disk 39 --vcpus 1 \${flavor}
  if ! openstack flavor list | grep \${flavor}; then
    echo -e "${red}ERROR: Unable to create flavor \${flavor}${reset}"
  fi
done
746
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" baremetal
747 748
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="control" control
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="compute" compute
749 750
echo "Configuring nameserver on ctlplane network"
neutron subnet-update \$(neutron subnet-list | grep -v id | grep -v \\\\-\\\\- | awk {'print \$2'}) --dns-nameserver 8.8.8.8
751
echo "Executing overcloud deployment, this should run for an extended period without output."
752
sleep 60 #wait for Hypervisor stats to check-in to nova
753
openstack overcloud deploy --templates $DEPLOY_OPTIONS
754 755 756 757 758
EOI

}

display_usage() {
Dan Radez's avatar
Dan Radez committed
759 760
  echo -e "Usage:\n$0 [arguments] \n"
  echo -e "   -c|--config : Directory to configuration files. Optional.  Defaults to /var/opt/opnfv/ \n"
761 762
  echo -e "   -d|--deploy-settings : Full path to deploy settings yaml file. Optional.  Defaults to null \n"
  echo -e "   -i|--inventory : Full path to inventory yaml file. Required only for baremetal \n"
763
  echo -e "   -n|--net-settings : Full path to network settings file. Optional. \n"
Dan Radez's avatar
Dan Radez committed
764 765 766
  echo -e "   -p|--ping-site : site to use to verify IP connectivity. Optional. Defaults to 8.8.8.8 \n"
  echo -e "   -r|--resources : Directory to deployment resources. Optional.  Defaults to /var/opt/opnfv/stack \n"
  echo -e "   -v|--virtual : Virtualize overcloud nodes instead of using baremetal. \n"
767
  echo -e "   --no-ha : disable High Availability deployment scheme, this assumes a single controller and single compute node \n"
768
  echo -e "   --flat : disable Network Isolation and use a single flat network for the underlay network."
769 770
}

771
##translates the command line parameters into variables
772 773 774 775 776 777 778
##params: $@ the entire command line is passed
##usage: parse_cmd_line() "$@"
parse_cmdline() {
  echo -e "\n\n${blue}This script is used to deploy the Apex Installer and Provision OPNFV Target System${reset}\n\n"
  echo "Use -h to display help"
  sleep 2

Dan Radez's avatar
Dan Radez committed
779
  while [ "${1:0:1}" = "-" ]
780 781
  do
    case "$1" in
782 783 784 785 786 787
        -h|--help)
                display_usage
                exit 0
            ;;
        -c|--config)
                CONFIG=$2
788
                echo "Deployment Configuration Directory Overridden to: $2"
789 790
                shift 2
            ;;
791 792 793 794 795 796 797
        -d|--deploy-settings)
                DEPLOY_SETTINGS_FILE=$2
                echo "Deployment Configuration file: $2"
                shift 2
            ;;
        -i|--inventory)
                INVENTORY_FILE=$2
798 799
                shift 2
            ;;
800 801 802
        -n|--net-settings)
                NETSETS=$2
                echo "Network Settings Configuration file: $2"
803 804 805 806
                shift 2
            ;;
        -p|--ping-site)
                ping_site=$2
807
                echo "Using $2 as the ping site"
808 809
                shift 2
            ;;
810 811
        -r|--resources)
                RESOURCES=$2
812
                echo "Deployment Resources Directory Overridden to: $2"
813
                shift 2
814
            ;;
815
        -v|--virtual)
816
                virtual="TRUE"
817
                echo "Executing a Virtual Deployment"
818 819
                shift 1
            ;;
820
        --no-ha )
821 822
                ha_enabled="FALSE"
                echo "HA Deployment Disabled"
Tim Rozet's avatar
Tim Rozet committed
823
                shift 1
824
            ;;
825
        --flat )
826 827
                net_isolation_enabled="FALSE"
                echo "Underlay Network Isolation Disabled: using flat configuration"
828 829
                shift 1
            ;;
830 831 832 833 834 835
        *)
                display_usage
                exit 1
            ;;
    esac
  done
836

837 838 839 840
  if [[ ! -z "$NETSETS" && "$net_isolation_enabled" == "FALSE" ]]; then
    echo -e "${red}INFO: Single flat network requested. Ignoring any network settings!${reset}"
  elif [[ -z "$NETSETS" && "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide a network_settings file with -n or use --flat to force a single flat network{reset}"
841 842 843 844 845 846 847 848 849 850 851 852
  fi

  if [[ -n "$virtual" && -n "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You should not specify an inventory with virtual deployments${reset}"
    exit 1
  fi

  if [[ ! -z "$DEPLOY_SETTINGS_FILE" && ! -f "$DEPLOY_SETTINGS_FILE" ]]; then
    echo -e "${red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

853 854
  if [[ ! -z "$NETSETS" && ! -f "$NETSETS" ]]; then
    echo -e "${red}ERROR: ${NETSETS} does not exist! Exiting...${reset}"
855 856 857 858 859 860 861 862 863 864 865 866
    exit 1
  fi

  if [[ ! -z "$INVENTORY_FILE" && ! -f "$INVENTORY_FILE" ]]; then
    echo -e "{$red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

  if [[ -z "$virtual" && -z "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You must specify an inventory file for baremetal deployments! Exiting...${reset}"
    exit 1
  fi
867 868 869 870 871 872
}

##END FUNCTIONS

main() {
  parse_cmdline "$@"
873 874 875 876
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${blue}INFO: Parsing network settings file...${reset}"
    parse_network_settings
  fi
Dan Radez's avatar
Dan Radez committed
877
  if ! configure_deps; then
878 879
    echo -e "${red}Dependency Validation Failed, Exiting.${reset}"
    exit 1
Dan Radez's avatar
Dan Radez committed
880
  fi
881 882 883
  if [ -n "$DEPLOY_SETTINGS_FILE" ]; then
    parse_deploy_settings
  fi
884
  setup_instack_vm
885
  if [ "$virtual" == "TRUE" ]; then
886
    setup_virtual_baremetal
887 888
  elif [ -n "$INVENTORY_FILE" ]; then
    parse_inventory_file
889
  fi
890
  configure_undercloud
891 892 893 894
  undercloud_prep_overcloud_deploy
}

main "$@"