deploy.sh 34.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
#!/bin/bash

# Deploy script to install provisioning server for OPNFV Apex
# author: Dan Radez (dradez@redhat.com)
# author: Tim Rozet (trozet@redhat.com)
#
# Based on RDO Manager http://www.rdoproject.org

set -e

##VARIABLES
12 13 14 15 16 17 18 19 20 21 22
if [ "$TERM" != "unknown" ]; then
  reset=$(tput sgr0)
  blue=$(tput setaf 4)
  red=$(tput setaf 1)
  green=$(tput setaf 2)
else
  reset=""
  blue=""
  red=""
  green=""
fi
23

Tim Rozet's avatar
Tim Rozet committed
24
vm_index=4
Dan Radez's avatar
Dan Radez committed
25
ha_enabled="TRUE"
Dan Radez's avatar
Dan Radez committed
26 27
ping_site="8.8.8.8"
ntp_server="pool.ntp.org"
28 29
net_isolation_enabled="TRUE"

30 31
declare -i CNT
declare UNDERCLOUD
32
declare -A deploy_options_array
33
declare -A NET_MAP
34

35
SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
36 37 38
DEPLOY_OPTIONS=""
RESOURCES=/var/opt/opnfv/stack
CONFIG=/var/opt/opnfv
39
INSTACKENV=$CONFIG/instackenv.json
40 41 42 43 44 45 46 47 48
OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
# Netmap used to map networks to OVS bridge names
NET_MAP['admin_network']="brbm"
NET_MAP['private_network']="brbm1"
NET_MAP['public_network']="brbm2"
NET_MAP['storage_network']="brbm3"

##LIBRARIES
source $CONFIG/lib/common-functions.sh
49

50
##FUNCTIONS
51 52 53 54 55 56 57 58 59 60 61 62 63 64
##translates yaml into variables
##params: filename, prefix (ex. "config_")
##usage: parse_yaml opnfv_ksgen_settings.yml "config_"
parse_yaml() {
   local prefix=$2
   local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
   sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  $1 |
   awk -F$fs '{
      indent = length($1)/2;
      vname[indent] = $2;
      for (i in vname) {if (i > indent) {delete vname[i]}}
      if (length($3) > 0) {
         vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
65
         printf("%s%s%s=%s\n", "'$prefix'",vn, $2, $3);
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
      }
   }'
}

##checks if prefix exists in string
##params: string, prefix
##usage: contains_prefix "deploy_setting_launcher=1" "deploy_setting"
contains_prefix() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^$prefix.*$" > /dev/null; then
    return 0
  else
    return 1
  fi
}
##parses variable from a string with '='
##and removes global prefix
##params: string, prefix
##usage: parse_setting_var 'deploy_myvar=2' 'deploy_'
parse_setting_var() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^.+\=" > /dev/null; then
    echo $(echo $mystr | grep -Eo "^.+\=" | tr -d '=' |  sed 's/^'"$prefix"'//')
  else
    return 1
  fi
}
##parses value from a string with '='
##params: string
##usage: parse_setting_value
parse_setting_value() {
  local mystr=$1
  echo $(echo $mystr | grep -Eo "\=.*$" | tr -d '=')
}
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
##parses network settings yaml into globals
parse_network_settings() {
  local required_network_settings="cidr"
  local common_optional_network_settings="usable_ip_range"
  local admin_network_optional_settings="provisioner_ip dhcp_range introspection_range"
  local public_network_optional_settings="floating_ip_range gateway provisioner_ip"
  local nic_value cidr

  eval $(parse_yaml ${NETSETS})
  for network in ${OPNFV_NETWORK_TYPES}; do
    if [[ $(eval echo \${${network}_enabled}) == 'true' ]]; then
      enabled_network_list+="${network} "
    elif [ "${network}" == 'admin_network' ]; then
      echo -e "${red}ERROR: You must enable admin_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    elif [ "${network}" == 'public_network' ]; then
      echo -e "${red}ERROR: You must enable public_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    else
      echo -e "${blue}INFO: Network: ${network} is disabled, will collapse into admin_network"
    fi
  done

  # check for enabled network values
  for enabled_network in ${enabled_network_list}; do
    # detect required settings first to continue
    echo -e "${blue}INFO: Detecting Required settings for: ${enabled_network}${reset}"
    for setting in ${required_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        # if setting is missing we try to autodetect
        eval "nic_value=\${${enabled_network}_bridged_interface}"
        if [ -n "$nic_value" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-detection failed: ${setting} not found using interface: ${nic_value}${reset}"
            exit 1
          fi
        else
          echo -e "${red}ERROR: Required setting: ${setting} not found, and bridge interface not provided\
for Auto-detection${reset}"
          exit 1
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Common settings for: ${enabled_network}${reset}"
    # detect optional common settings
    # these settings can be auto-generated if missing
    for setting in ${common_optional_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        setting_value=$(eval find_${setting} ${nic_value})
        if [ -n "$setting_value" ]; then
          eval "${enabled_network}_${setting}=${setting_value}"
          echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
        else
          # if Auto-detection fails we can auto-generate with CIDR
          eval "cidr=\${${enabled_network}_cidr}"
          setting_value=$(eval generate_${setting} ${cidr})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
            exit 1
          fi
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Network Specific settings for: ${enabled_network}${reset}"
    # detect network specific settings
    if [ -n $(eval echo \${${network}_optional_settings}) ]; then
      eval "network_specific_settings=\${${enabled_network}_optional_settings}"
      for setting in ${network_specific_settings}; do
        eval "setting_value=\${${enabled_network}_${setting}}"
        if [ -z "${setting_value}" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            eval "cidr=\${${enabled_network}_cidr}"
            setting_value=$(eval generate_${setting} ${cidr})
            if [ -n "$setting_value" ]; then
              eval "${enabled_network}_${setting}=${setting_value}"
              echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
            else
              echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
              exit 1
            fi
          fi
        else
          echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
        fi
      done
    fi
  done
}
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
##parses deploy settings yaml into globals and options array
##params: none
##usage:  parse_deploy_settings
parse_deploy_settings() {
  local global_prefix="deploy_global_params_"
  local options_prefix="deploy_deploy_options_"
  local myvar myvalue
  local settings=$(parse_yaml $DEPLOY_SETTINGS_FILE "deploy_")

  for this_setting in $settings; do
    if contains_prefix $this_setting $global_prefix; then
      myvar=$(parse_setting_var $this_setting $global_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      # Do not override variables set by cmdline
      if [ -z "$(eval echo \$$myvar)" ]; then
        eval "$myvar=\$myvalue"
        echo -e "${blue}Global parameter set: ${myvar}:${myvalue}${reset}"
      else
        echo -e "${blue}Global parameter already set: ${myvar}${reset}"
      fi
    elif contains_prefix $this_setting $options_prefix; then
      myvar=$(parse_setting_var $this_setting $options_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      deploy_options_array[$myvar]=$myvalue
      echo -e "${blue}Deploy option set: ${myvar}:${myvalue}${reset}"
    fi
  done
}
##parses baremetal yaml settings into compatible json
##writes the json to $CONFIG/instackenv_tmp.json
##params: none
##usage: parse_inventory_file
parse_inventory_file() {
  local inventory=$(parse_yaml $INVENTORY_FILE)
  local node_list
  local node_prefix="node"
  local node_count=0
  local node_total
  local inventory_list

  # detect number of nodes
  for entry in $inventory; do
    if echo $entry | grep -Eo "^nodes_node[0-9]+_" > /dev/null; then
      this_node=$(echo $entry | grep -Eo "^nodes_node[0-9]+_")
      if [[ $inventory_list != *"$this_node"* ]]; then
        inventory_list+="$this_node "
      fi
    fi
  done

  inventory_list=$(echo $inventory_list | sed 's/ $//')

  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
266
    ((node_count+=1))
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
  done

  node_total=$node_count

  if [[ "$node_total" -lt 5 && ha_enabled == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide at least 5 nodes for HA baremetal deployment${reset}"
    exit 1
  elif [[ "$node_total" -lt 2 ]]; then
    echo -e "${red}ERROR: You must provide at least 2 nodes for non-HA baremetal deployment${reset}"
    exit 1
  fi

  eval $(parse_yaml $INVENTORY_FILE)

  instack_env_output="
{
 \"nodes\" : [

"
  node_count=0
  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
288 289
    ((node_count+=1))
    node_output="
290 291 292 293 294 295 296 297 298 299 300
        {
          \"pm_password\": \"$(eval echo \${${node}ipmi_pass})\",
          \"pm_type\": \"pxe_ipmitool\",
          \"mac\": [
            \"$(eval echo \${${node}mac_address})\"
          ],
          \"cpu\": \"$(eval echo \${${node}cpus})\",
          \"memory\": \"$(eval echo \${${node}memory})\",
          \"disk\": \"$(eval echo \${${node}disk})\",
          \"arch\": \"$(eval echo \${${node}arch})\",
          \"pm_user\": \"$(eval echo \${${node}ipmi_user})\",
301 302
          \"pm_addr\": \"$(eval echo \${${node}ipmi_ip})\",
          \"capabilities\": \"$(eval echo \${${node}capabilities})\"
303
"
Dan Radez's avatar
Dan Radez committed
304
    instack_env_output+=${node_output}
305 306 307 308 309 310 311 312 313 314 315 316 317
    if [ $node_count -lt $node_total ]; then
      instack_env_output+="        },"
    else
      instack_env_output+="        }"
    fi
  done

  instack_env_output+='
  ]
}
'
  #Copy instackenv.json to undercloud for baremetal
  echo -e "{blue}Parsed instackenv JSON:\n${instack_env_output}${reset}"
Dan Radez's avatar
Dan Radez committed
318
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
319 320 321 322 323 324
cat > instackenv.json << EOF
$instack_env_output
EOF
EOI

}
325 326 327
##verify internet connectivity
#params: none
function verify_internet {
Dan Radez's avatar
Dan Radez committed
328
  if ping -c 2 $ping_site > /dev/null; then
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
    if ping -c 2 www.google.com > /dev/null; then
      echo "${blue}Internet connectivity detected${reset}"
      return 0
    else
      echo "${red}Internet connectivity detected, but DNS lookup failed${reset}"
      return 1
    fi
  else
    echo "${red}No internet connectivity detected${reset}"
    return 1
  fi
}

##download dependencies if missing and configure host
#params: none
function configure_deps {
  if ! verify_internet; then
    echo "${red}Will not download dependencies${reset}"
    internet=false
Dan Radez's avatar
Dan Radez committed
348 349 350 351 352 353
  fi

  # verify ip forwarding
  if sysctl net.ipv4.ip_forward | grep 0; then
    sudo sysctl -w net.ipv4.ip_forward=1
    sudo sh -c "echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf"
354 355
  fi

356 357 358 359 360 361 362 363
  # ensure no dhcp server is running on jumphost
  if ! sudo systemctl status dhcpd | grep dead; then
    echo "${red}WARN: DHCP Server detected on jumphost, disabling...${reset}"
    sudo systemctl stop dhcpd
    sudo systemctl disable dhcpd
  fi

  # ensure networks are configured
364
  systemctl start openvswitch
365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391

  # If flat we only use admin network
  if [[ "$net_isolation_enabled" == "FALSE" ]]; then
    virsh_enabled_networks="admin_network"
  # For baremetal we only need to create/attach instack to admin and public
  elif [ "$virtual" == "FALSE" ]; then
    virsh_enabled_networks="admin_network public_network"
  else
    virsh_enabled_neworks=$enabled_network_list
  fi

  for network in ${OPNFV_NETWORK_TYPES}; do
    ovs-vsctl list-br | grep ${NET_MAP[$network]} > /dev/null || ovs-vsctl add-br ${NET_MAP[$network]}
    virsh net-list --all | grep ${NET_MAP[$network]} > /dev/null || virsh net-create $CONFIG/${NET_MAP[$network]}-net.xml
    virsh net-list | grep -E "${NET_MAP[$network]}\s+active" > /dev/null || virsh net-start ${NET_MAP[$network]}
  done

  echo -e "${blue}INFO: Bridges set: ${reset}"
  ovs-vsctl list-br
  echo -e "${blue}INFO: virsh networks set: ${reset}"
  virsh net-list

  if [[ -z "$virtual" || "$virtual" == "FALSE" ]]; then
    # bridge interfaces to correct OVS instances for baremetal deployment
    for network in ${enabled_network_list}; do
      this_interface=$(eval echo \${${network}_bridged_interface})
      # check if this a bridged interface for this network
392 393 394 395 396 397 398
      if [[ ! -z "$this_interface" || "$this_interface" != "none" ]]; then
        if ! attach_interface_to_ovs ${NET_MAP[$network]} ${this_interface} ${network}; then
          echo -e "${red}ERROR: Unable to bridge interface ${this_interface} to bridge ${NET_MAP[$network]} for enabled network: ${network}${reset}"
          exit 1
        else
          echo -e "${blue}INFO: Interface ${this_interface} bridged to bridge ${NET_MAP[$network]} for enabled network: ${network}${reset}"
        fi
399 400 401 402 403 404
      else
        echo "${red}ERROR: Unable to determine interface to bridge to for enabled network: ${network}${reset}"
        exit 1
      fi
    done
  fi
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427

  # ensure storage pool exists and is started
  virsh pool-list --all | grep default > /dev/null || virsh pool-create $CONFIG/default-pool.xml
  virsh pool-list | grep -Eo "default\s+active" > /dev/null || virsh pool-start default

  if virsh net-list | grep default > /dev/null; then
    num_ints_same_subnet=$(ip addr show | grep "inet 192.168.122" | wc -l)
    if [ "$num_ints_same_subnet" -gt 1 ]; then
      virsh net-destroy default
      ##go edit /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' instackenv-virt.json
      sleep 5
      virsh net-start default
      virsh net-autostart default
    fi
  fi

  if ! egrep '^flags.*(vmx|svm)' /proc/cpuinfo > /dev/null; then
    echo "${red}virtualization extensions not found, kvm kernel module insertion may fail.\n  \
Are you sure you have enabled vmx in your bios or hypervisor?${reset}"
  fi

Dan Radez's avatar
Dan Radez committed
428 429
  if ! lsmod | grep kvm > /dev/null; then modprobe kvm; fi
  if ! lsmod | grep kvm_intel > /dev/null; then modprobe kvm_intel; fi
430 431 432 433 434 435 436 437 438 439 440 441 442

  if ! lsmod | grep kvm > /dev/null; then
    echo "${red}kvm kernel modules not loaded!${reset}"
    return 1
  fi

  ##sshkeygen for root
  if [ ! -e ~/.ssh/id_rsa.pub ]; then
    ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
  fi

  echo "${blue}All dependencies installed and running${reset}"
}
443 444

##verify vm exists, an has a dhcp lease assigned to it
Dan Radez's avatar
Dan Radez committed
445
##params: none
446
function setup_instack_vm {
447
  if ! virsh list --all | grep instack > /dev/null; then
448
      #virsh vol-create default instack.qcow2.xml
449
      virsh define $CONFIG/instack.xml
450

451 452 453 454
      #Upload instack image
      #virsh vol-create default --file instack.qcow2.xml
      virsh vol-create-as default instack.qcow2 30G --format qcow2

455 456 457 458 459 460 461 462 463
      ### this doesn't work for some reason I was getting hangup events so using cp instead
      #virsh vol-upload --pool default --vol instack.qcow2 --file $CONFIG/stack/instack.qcow2
      #2015-12-05 12:57:20.569+0000: 8755: info : libvirt version: 1.2.8, package: 16.el7_1.5 (CentOS BuildSystem <http://bugs.centos.org>, 2015-11-03-13:56:46, worker1.bsys.centos.org)
      #2015-12-05 12:57:20.569+0000: 8755: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #2015-12-05 12:57:20.569+0000: 8756: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #error: cannot close volume instack.qcow2
      #error: internal error: received hangup / error event on socket
      #error: Reconnected to the hypervisor

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
      instack_dst=/var/lib/libvirt/images/instack.qcow2
      cp -f $RESOURCES/instack.qcow2 $instack_dst

      # resize instack machine
      echo "Checking if instack needs to be resized..."
      instack_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep device | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
      if [ "$instack_size" -lt 30 ]; then
        qemu-img resize /var/lib/libvirt/images/instack.qcow2 +25G
	LIBGUESTFS_BACKEND=direct virt-resize --expand /dev/sda1 $RESOURCES/instack.qcow2 $instack_dst
	LIBGUESTFS_BACKEND=direct virt-customize -a $instack_dst --run-command 'xfs_growfs -d /dev/sda1 || true'
        new_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep filesystem | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
        if [ "$new_size" -lt 30 ]; then
          echo "Error resizing instack machine, disk size is ${new_size}"
          exit 1
        else
          echo "instack successfully resized"
        fi
      else
        echo "skipped instack resize, upstream is large enough"
      fi
484 485 486 487

  else
      echo "Found Instack VM, using existing VM"
  fi
488 489 490 491

  # if the VM is not running update the authkeys and start it
  if ! virsh list | grep instack > /dev/null; then
    echo "Injecting ssh key to instack VM"
492 493
    virt-customize -c qemu:///system -d instack --run-command "mkdir /root/.ssh/" \
        --upload ~/.ssh/id_rsa.pub:/root/.ssh/authorized_keys \
494 495 496 497 498 499
        --run-command "chmod 600 /root/.ssh/authorized_keys && restorecon /root/.ssh/authorized_keys" \
        --run-command "cp /root/.ssh/authorized_keys /home/stack/.ssh/" \
        --run-command "chown stack:stack /home/stack/.ssh/authorized_keys && chmod 600 /home/stack/.ssh/authorized_keys"
    virsh start instack
  fi

500 501 502
  sleep 3 # let DHCP happen

  CNT=10
503 504
  echo -n "${blue}Waiting for instack's dhcp address${reset}"
  while ! grep instack /var/lib/libvirt/dnsmasq/default.leases > /dev/null && [ $CNT -gt 0 ]; do
505 506 507 508 509 510
      echo -n "."
      sleep 3
      CNT=CNT-1
  done

  # get the instack VM IP
511
  UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
Dan Radez's avatar
Dan Radez committed
512
  if [ -z "$UNDERCLOUD" ]; then
513 514 515
    #if not found then dnsmasq may be using leasefile-ro
    instack_mac=$(virsh domiflist instack | grep default | \
                  grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
516
    UNDERCLOUD=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
517 518 519 520

    if [ -z "$UNDERCLOUD" ]; then
      echo "\n\nNever got IP for Instack. Can Not Continue."
      exit 1
521 522
    else
      echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
523
    fi
Dan Radez's avatar
Dan Radez committed
524 525 526
  else
     echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
  fi
527 528

  CNT=10
529
  echo -en "${blue}\rValidating instack VM connectivity${reset}"
530 531 532
  while ! ping -c 1 $UNDERCLOUD > /dev/null && [ $CNT -gt 0 ]; do
      echo -n "."
      sleep 3
533
      CNT=$CNT-1
534
  done
Dan Radez's avatar
Dan Radez committed
535
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
536 537 538
      echo "Failed to contact Instack. Can Not Continue"
      exit 1
  fi
539
  CNT=10
540
  while ! ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "echo ''" 2>&1> /dev/null && [ $CNT -gt 0 ]; do
541 542
      echo -n "."
      sleep 3
543
      CNT=$CNT-1
544
  done
Dan Radez's avatar
Dan Radez committed
545
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
546 547 548
      echo "Failed to connect to Instack. Can Not Continue"
      exit 1
  fi
549 550

  # extra space to overwrite the previous connectivity output
Dan Radez's avatar
Dan Radez committed
551
  echo -e "${blue}\r                                                                 ${reset}"
552

553 554 555 556 557 558
  #add the instack public interface if net isolation is enabled (more than just admin network)
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    virsh attach-interface --domain instack --type network --source ${NET_MAP['public_network']} --model rtl8139 --config --live
    sleep 1
    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "if ! ip a s eth2 | grep ${public_network_provisioner_ip} > /dev/null; then ip a a ${public_network_provisioner_ip}/${public_network_cidr##*/} dev eth2; ip link set up dev eth2; fi"
  fi
559 560 561
  # ssh key fix for stack user
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "restorecon -r /home/stack"
}
562

Tim Rozet's avatar
Tim Rozet committed
563 564
##Create virtual nodes in virsh
##params: none
565 566
function setup_virtual_baremetal {
  for i in $(seq 0 $vm_index); do
567 568 569
    if ! virsh list --all | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} > /dev/null; then
      if [ ! -e $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml ]; then
        define_virtual_node baremetalbrbm_brbm1_brbm2_brbm3_${i}
Tim Rozet's avatar
Tim Rozet committed
570
      fi
571 572 573 574 575 576 577
      # Fix for ramdisk using wrong pxeboot interface
      # TODO: revisit this and see if there's a more proper fix
      sed -i "/^\s*<source network='brbm2'\/>/{
        N
        s/^\(.*\)virtio\(.*\)$/\1rtl8139\2/
        }" $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
      virsh define $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
578 579 580
    else
      echo "Found Baremetal ${i} VM, using existing VM"
    fi
581
    virsh vol-list default | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} 2>&1> /dev/null || virsh vol-create-as default baremetalbrbm_brbm1_brbm2_brbm3_${i}.qcow2 40G --format qcow2
582
  done
583

584 585
}

586 587 588 589 590 591 592 593
##Set network-environment settings
##params: network-environment file to edit
function configure_network_environment {
  sed -i '/ControlPlaneSubnetCidr/c\\  ControlPlaneSubnetCidr: "'${admin_network_cidr##*/}'"' $1
  sed -i '/ControlPlaneDefaultRoute/c\\  ControlPlaneDefaultRoute: '${admin_network_provisioner_ip}'' $1
  sed -i '/ExternalNetCidr/c\\  ExternalNetCidr: '${public_network_cidr}'' $1
  sed -i "/ExternalAllocationPools/c\\  ExternalAllocationPools: [{'start': '${public_network_usable_ip_range%%,*}', 'end': '${public_network_usable_ip_range##*,}'}]" $1
  sed -i '/ExternalInterfaceDefaultRoute/c\\  ExternalInterfaceDefaultRoute: '${public_network_gateway}'' $1
594
  sed -i '/EC2MetadataIp/c\\  EC2MetadataIp: '${admin_network_provisioner_ip}'' $1
595
}
596
##Copy over the glance images and instack json file
Tim Rozet's avatar
Tim Rozet committed
597
##params: none
598
function configure_undercloud {
599

600 601
  echo
  echo "Copying configuration file and disk images to instack"
602
  scp ${SSH_OPTIONS[@]} $RESOURCES/overcloud-full.qcow2 "stack@$UNDERCLOUD":
603 604 605 606 607 608
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    configure_network_environment $CONFIG/network-environment.yaml
    echo -e "${blue}Network Environment set for Deployment: ${reset}"
    cat $CONFIG/network-environment.yaml
    scp ${SSH_OPTIONS[@]} $CONFIG/network-environment.yaml "stack@$UNDERCLOUD":
  fi
Dan Radez's avatar
Dan Radez committed
609
  scp ${SSH_OPTIONS[@]} -r $CONFIG/nics/ "stack@$UNDERCLOUD":
610

611 612 613
  # ensure stack user on instack machine has an ssh key
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "if [ ! -e ~/.ssh/id_rsa.pub ]; then ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa; fi"

614
  if [ "$virtual" == "TRUE" ]; then
615 616 617 618 619 620

      # copy the instack vm's stack user's pub key to
      # root's auth keys so that instack can control
      # vm power on the hypervisor
      ssh ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys

621 622 623 624 625 626 627
      # fix MACs to match new setup
      for i in $(seq 0 $vm_index); do
        pyscript="import json
data = json.load(open('$CONFIG/instackenv-virt.json'))
print data['nodes'][$i]['mac'][0]"

        old_mac=$(python -c "$pyscript")
628
        new_mac=$(virsh dumpxml baremetalbrbm_brbm1_brbm2_brbm3_$i | grep "mac address" | cut -d = -f2 | grep -Eo "[0-9a-f:]+")
629
        # this doesn't work with multiple vnics on the vms
Dan Radez's avatar
Dan Radez committed
630 631 632 633
        #if [ "$old_mac" != "$new_mac" ]; then
        #  echo "${blue}Modifying MAC for node from $old_mac to ${new_mac}${reset}"
        #  sed -i 's/'"$old_mac"'/'"$new_mac"'/' $CONFIG/instackenv-virt.json
        #fi
634 635
      done

636
      DEPLOY_OPTIONS+=" --libvirt-type qemu"
637
      INSTACKENV=$CONFIG/instackenv-virt.json
638

639 640 641
      # upload instackenv file to Instack for virtual deployment
      scp ${SSH_OPTIONS[@]} $INSTACKENV "stack@$UNDERCLOUD":instackenv.json
  fi
642 643 644

  # allow stack to control power management on the hypervisor via sshkey
  # only if this is a virtual deployment
645
  if [ "$virtual" == "TRUE" ]; then
646 647 648 649 650 651 652 653 654
      ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
while read -r line; do
  stack_key=\${stack_key}\\\\\\\\n\${line}
done < <(cat ~/.ssh/id_rsa)
stack_key=\$(echo \$stack_key | sed 's/\\\\\\\\n//')
sed -i 's~INSERT_STACK_USER_PRIV_KEY~'"\$stack_key"'~' instackenv.json
EOI
  fi

655 656 657
  # copy stack's ssh key to this users authorized keys
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> ~/.ssh/authorized_keys

658 659 660
  # disable requiretty for sudo
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "sed -i 's/Defaults\s*requiretty//'" /etc/sudoers

661 662 663 664
  # configure undercloud on Undercloud VM
  echo "Running undercloud configuration."
  echo "Logging undercloud configuration to instack:/home/stack/apex-undercloud-install.log"
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" << EOI
665
if [[ "$net_isolation_enabled" == "TRUE" ]]; then
666 667 668 669 670 671 672 673
  sed -i 's/#local_ip/local_ip/' undercloud.conf
  sed -i 's/#network_gateway/network_gateway/' undercloud.conf
  sed -i 's/#network_cidr/network_cidr/' undercloud.conf
  sed -i 's/#dhcp_start/dhcp_start/' undercloud.conf
  sed -i 's/#dhcp_end/dhcp_end/' undercloud.conf
  sed -i 's/#inspection_iprange/inspection_iprange/' undercloud.conf
  sed -i 's/#undercloud_debug/undercloud_debug/' undercloud.conf

674 675 676 677 678 679
  openstack-config --set undercloud.conf DEFAULT local_ip ${admin_network_provisioner_ip}/${admin_network_cidr##*/}
  openstack-config --set undercloud.conf DEFAULT network_gateway ${admin_network_provisioner_ip}
  openstack-config --set undercloud.conf DEFAULT network_cidr ${admin_network_cidr}
  openstack-config --set undercloud.conf DEFAULT dhcp_start ${admin_network_dhcp_range%%,*}
  openstack-config --set undercloud.conf DEFAULT dhcp_end ${admin_network_dhcp_range##*,}
  openstack-config --set undercloud.conf DEFAULT inspection_iprange ${admin_network_introspection_range}
680 681 682 683
  openstack-config --set undercloud.conf DEFAULT undercloud_debug false
fi

openstack undercloud install &> apex-undercloud-install.log
684 685 686 687
sleep 30
sudo systemctl restart openstack-glance-api
sudo systemctl restart openstack-nova-conductor
sudo systemctl restart openstack-nova-compute
688
EOI
689 690 691 692 693 694 695
# WORKAROUND: must restart the above services to fix sync problem with nova compute manager
# TODO: revisit and file a bug if necessary. This should eventually be removed
# as well as glance api problem
echo -e "${blue}INFO: Sleeping 15 seconds while services come back from restart${reset}"
sleep 15
#TODO Fill in the rest of the network-environment values for other networks

696 697 698
}

##preping it for deployment and launch the deploy
Tim Rozet's avatar
Tim Rozet committed
699
##params: none
700
function undercloud_prep_overcloud_deploy {
701 702 703 704 705 706

  if [[ ${#deploy_options_array[@]} -eq 0 || ${deploy_options_array['sdn_controller']} == 'opendaylight' ]]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'opendaylight-external' ]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight-external.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'onos' ]; then
bobzh's avatar
bobzh committed
707
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/onos.yaml"
708 709 710 711
  elif [ ${deploy_options_array['sdn_controller']} == 'opencontrail' ]; then
    echo -e "${red}ERROR: OpenContrail is currently unsupported...exiting${reset}"
    exit 1
  fi
712

Tim Rozet's avatar
Tim Rozet committed
713
  # check if HA is enabled
714
  if [[ "$ha_enabled" == "TRUE" ]]; then
Tim Rozet's avatar
Tim Rozet committed
715 716 717
     DEPLOY_OPTIONS+=" --control-scale 3 --compute-scale 2"
     DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/puppet-pacemaker.yaml"
  fi
718

719
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
720
     #DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml"
721
     DEPLOY_OPTIONS+=" -e network-environment.yaml"
722 723
  fi

724
  if [[ "$ha_enabled" == "TRUE" ]] || [[ "$net_isolation_enabled" == "TRUE" ]]; then
Dan Radez's avatar
Dan Radez committed
725
     DEPLOY_OPTIONS+=" --ntp-server $ntp_server"
726 727
  fi

728 729 730 731
  if [[ ! "$virtual" == "TRUE" ]]; then
     DEPLOY_OPTIONS+=" --control-flavor control --compute-flavor compute"
  fi

732
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
733
source stackrc
734 735
set -o errexit
echo "Uploading overcloud glance images"
736
openstack overcloud image upload
737
echo "Configuring undercloud and discovering nodes"
738 739 740
openstack baremetal import --json instackenv.json
openstack baremetal configure boot
openstack baremetal introspection bulk start
741
echo "Configuring flavors"
742 743 744 745 746 747 748 749 750 751
for flavor in baremetal control compute; do
  echo -e "${blue}INFO: Updating flavor: \${flavor}${reset}"
  if openstack flavor list | grep \${flavor}; then
    openstack flavor delete \${flavor}
  fi
  openstack flavor create --id auto --ram 4096 --disk 39 --vcpus 1 \${flavor}
  if ! openstack flavor list | grep \${flavor}; then
    echo -e "${red}ERROR: Unable to create flavor \${flavor}${reset}"
  fi
done
752
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" baremetal
753 754
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="control" control
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="compute" compute
755 756
echo "Configuring nameserver on ctlplane network"
neutron subnet-update \$(neutron subnet-list | grep -v id | grep -v \\\\-\\\\- | awk {'print \$2'}) --dns-nameserver 8.8.8.8
757
echo "Executing overcloud deployment, this should run for an extended period without output."
758
sleep 60 #wait for Hypervisor stats to check-in to nova
759 760 761 762
# save deploy command so it can be used for debugging
cat > deploy_command << EOF
openstack overcloud deploy --templates $DEPLOY_OPTIONS
EOF
763
openstack overcloud deploy --templates $DEPLOY_OPTIONS
764 765 766 767 768
EOI

}

display_usage() {
Dan Radez's avatar
Dan Radez committed
769 770
  echo -e "Usage:\n$0 [arguments] \n"
  echo -e "   -c|--config : Directory to configuration files. Optional.  Defaults to /var/opt/opnfv/ \n"
771 772
  echo -e "   -d|--deploy-settings : Full path to deploy settings yaml file. Optional.  Defaults to null \n"
  echo -e "   -i|--inventory : Full path to inventory yaml file. Required only for baremetal \n"
773
  echo -e "   -n|--net-settings : Full path to network settings file. Optional. \n"
Dan Radez's avatar
Dan Radez committed
774 775 776
  echo -e "   -p|--ping-site : site to use to verify IP connectivity. Optional. Defaults to 8.8.8.8 \n"
  echo -e "   -r|--resources : Directory to deployment resources. Optional.  Defaults to /var/opt/opnfv/stack \n"
  echo -e "   -v|--virtual : Virtualize overcloud nodes instead of using baremetal. \n"
777
  echo -e "   --no-ha : disable High Availability deployment scheme, this assumes a single controller and single compute node \n"
778
  echo -e "   --flat : disable Network Isolation and use a single flat network for the underlay network."
779 780
}

781
##translates the command line parameters into variables
782 783 784 785 786 787 788
##params: $@ the entire command line is passed
##usage: parse_cmd_line() "$@"
parse_cmdline() {
  echo -e "\n\n${blue}This script is used to deploy the Apex Installer and Provision OPNFV Target System${reset}\n\n"
  echo "Use -h to display help"
  sleep 2

Dan Radez's avatar
Dan Radez committed
789
  while [ "${1:0:1}" = "-" ]
790 791
  do
    case "$1" in
792 793 794 795 796 797
        -h|--help)
                display_usage
                exit 0
            ;;
        -c|--config)
                CONFIG=$2
798
                echo "Deployment Configuration Directory Overridden to: $2"
799 800
                shift 2
            ;;
801 802 803 804 805 806 807
        -d|--deploy-settings)
                DEPLOY_SETTINGS_FILE=$2
                echo "Deployment Configuration file: $2"
                shift 2
            ;;
        -i|--inventory)
                INVENTORY_FILE=$2
808 809
                shift 2
            ;;
810 811 812
        -n|--net-settings)
                NETSETS=$2
                echo "Network Settings Configuration file: $2"
813 814 815 816
                shift 2
            ;;
        -p|--ping-site)
                ping_site=$2
817
                echo "Using $2 as the ping site"
818 819
                shift 2
            ;;
820 821
        -r|--resources)
                RESOURCES=$2
822
                echo "Deployment Resources Directory Overridden to: $2"
823
                shift 2
824
            ;;
825
        -v|--virtual)
826
                virtual="TRUE"
827
                echo "Executing a Virtual Deployment"
828 829
                shift 1
            ;;
830
        --no-ha )
831 832
                ha_enabled="FALSE"
                echo "HA Deployment Disabled"
Tim Rozet's avatar
Tim Rozet committed
833
                shift 1
834
            ;;
835
        --flat )
836 837
                net_isolation_enabled="FALSE"
                echo "Underlay Network Isolation Disabled: using flat configuration"
838 839
                shift 1
            ;;
840 841 842 843 844 845
        *)
                display_usage
                exit 1
            ;;
    esac
  done
846

847 848 849 850
  if [[ ! -z "$NETSETS" && "$net_isolation_enabled" == "FALSE" ]]; then
    echo -e "${red}INFO: Single flat network requested. Ignoring any network settings!${reset}"
  elif [[ -z "$NETSETS" && "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide a network_settings file with -n or use --flat to force a single flat network{reset}"
851 852 853 854 855 856 857 858 859 860 861 862
  fi

  if [[ -n "$virtual" && -n "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You should not specify an inventory with virtual deployments${reset}"
    exit 1
  fi

  if [[ ! -z "$DEPLOY_SETTINGS_FILE" && ! -f "$DEPLOY_SETTINGS_FILE" ]]; then
    echo -e "${red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

863 864
  if [[ ! -z "$NETSETS" && ! -f "$NETSETS" ]]; then
    echo -e "${red}ERROR: ${NETSETS} does not exist! Exiting...${reset}"
865 866 867 868 869 870 871 872 873 874 875 876
    exit 1
  fi

  if [[ ! -z "$INVENTORY_FILE" && ! -f "$INVENTORY_FILE" ]]; then
    echo -e "{$red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

  if [[ -z "$virtual" && -z "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You must specify an inventory file for baremetal deployments! Exiting...${reset}"
    exit 1
  fi
877 878 879 880 881 882
}

##END FUNCTIONS

main() {
  parse_cmdline "$@"
883 884 885 886
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${blue}INFO: Parsing network settings file...${reset}"
    parse_network_settings
  fi
Dan Radez's avatar
Dan Radez committed
887
  if ! configure_deps; then
888 889
    echo -e "${red}Dependency Validation Failed, Exiting.${reset}"
    exit 1
Dan Radez's avatar
Dan Radez committed
890
  fi
891 892 893
  if [ -n "$DEPLOY_SETTINGS_FILE" ]; then
    parse_deploy_settings
  fi
894
  setup_instack_vm
895
  if [ "$virtual" == "TRUE" ]; then
896
    setup_virtual_baremetal
897 898
  elif [ -n "$INVENTORY_FILE" ]; then
    parse_inventory_file
899
  fi
900
  configure_undercloud
901 902 903 904
  undercloud_prep_overcloud_deploy
}

main "$@"