deploy.sh 34.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
#!/bin/bash

# Deploy script to install provisioning server for OPNFV Apex
# author: Dan Radez (dradez@redhat.com)
# author: Tim Rozet (trozet@redhat.com)
#
# Based on RDO Manager http://www.rdoproject.org

set -e

##VARIABLES
12
13
14
15
16
17
18
19
20
21
22
if [ "$TERM" != "unknown" ]; then
  reset=$(tput sgr0)
  blue=$(tput setaf 4)
  red=$(tput setaf 1)
  green=$(tput setaf 2)
else
  reset=""
  blue=""
  red=""
  green=""
fi
23

Tim Rozet's avatar
Tim Rozet committed
24
vm_index=4
Dan Radez's avatar
Dan Radez committed
25
ha_enabled="TRUE"
Dan Radez's avatar
Dan Radez committed
26
27
ping_site="8.8.8.8"
ntp_server="pool.ntp.org"
28
29
net_isolation_enabled="TRUE"

30
31
declare -i CNT
declare UNDERCLOUD
32
declare -A deploy_options_array
33
declare -A NET_MAP
34

35
SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
36
37
38
DEPLOY_OPTIONS=""
RESOURCES=/var/opt/opnfv/stack
CONFIG=/var/opt/opnfv
39
INSTACKENV=$CONFIG/instackenv.json
40
41
42
43
44
45
46
47
48
OPNFV_NETWORK_TYPES="admin_network private_network public_network storage_network"
# Netmap used to map networks to OVS bridge names
NET_MAP['admin_network']="brbm"
NET_MAP['private_network']="brbm1"
NET_MAP['public_network']="brbm2"
NET_MAP['storage_network']="brbm3"

##LIBRARIES
source $CONFIG/lib/common-functions.sh
49

50
##FUNCTIONS
51
52
53
54
55
56
57
58
59
60
61
62
63
64
##translates yaml into variables
##params: filename, prefix (ex. "config_")
##usage: parse_yaml opnfv_ksgen_settings.yml "config_"
parse_yaml() {
   local prefix=$2
   local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
   sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
        -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p"  $1 |
   awk -F$fs '{
      indent = length($1)/2;
      vname[indent] = $2;
      for (i in vname) {if (i > indent) {delete vname[i]}}
      if (length($3) > 0) {
         vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
65
         printf("%s%s%s=%s\n", "'$prefix'",vn, $2, $3);
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
      }
   }'
}

##checks if prefix exists in string
##params: string, prefix
##usage: contains_prefix "deploy_setting_launcher=1" "deploy_setting"
contains_prefix() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^$prefix.*$" > /dev/null; then
    return 0
  else
    return 1
  fi
}
##parses variable from a string with '='
##and removes global prefix
##params: string, prefix
##usage: parse_setting_var 'deploy_myvar=2' 'deploy_'
parse_setting_var() {
  local mystr=$1
  local prefix=$2
  if echo $mystr | grep -E "^.+\=" > /dev/null; then
    echo $(echo $mystr | grep -Eo "^.+\=" | tr -d '=' |  sed 's/^'"$prefix"'//')
  else
    return 1
  fi
}
##parses value from a string with '='
##params: string
##usage: parse_setting_value
parse_setting_value() {
  local mystr=$1
  echo $(echo $mystr | grep -Eo "\=.*$" | tr -d '=')
}
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
##parses network settings yaml into globals
parse_network_settings() {
  local required_network_settings="cidr"
  local common_optional_network_settings="usable_ip_range"
  local admin_network_optional_settings="provisioner_ip dhcp_range introspection_range"
  local public_network_optional_settings="floating_ip_range gateway provisioner_ip"
  local nic_value cidr

  eval $(parse_yaml ${NETSETS})
  for network in ${OPNFV_NETWORK_TYPES}; do
    if [[ $(eval echo \${${network}_enabled}) == 'true' ]]; then
      enabled_network_list+="${network} "
    elif [ "${network}" == 'admin_network' ]; then
      echo -e "${red}ERROR: You must enable admin_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    elif [ "${network}" == 'public_network' ]; then
      echo -e "${red}ERROR: You must enable public_network and configure it explicitly or use auto-detection${reset}"
      exit 1
    else
      echo -e "${blue}INFO: Network: ${network} is disabled, will collapse into admin_network"
    fi
  done

  # check for enabled network values
  for enabled_network in ${enabled_network_list}; do
    # detect required settings first to continue
    echo -e "${blue}INFO: Detecting Required settings for: ${enabled_network}${reset}"
    for setting in ${required_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        # if setting is missing we try to autodetect
        eval "nic_value=\${${enabled_network}_bridged_interface}"
        if [ -n "$nic_value" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-detection failed: ${setting} not found using interface: ${nic_value}${reset}"
            exit 1
          fi
        else
          echo -e "${red}ERROR: Required setting: ${setting} not found, and bridge interface not provided\
for Auto-detection${reset}"
          exit 1
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Common settings for: ${enabled_network}${reset}"
    # detect optional common settings
    # these settings can be auto-generated if missing
    for setting in ${common_optional_network_settings}; do
      eval "setting_value=\${${enabled_network}_${setting}}"
      if [ -z "${setting_value}" ]; then
        setting_value=$(eval find_${setting} ${nic_value})
        if [ -n "$setting_value" ]; then
          eval "${enabled_network}_${setting}=${setting_value}"
          echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
        else
          # if Auto-detection fails we can auto-generate with CIDR
          eval "cidr=\${${enabled_network}_cidr}"
          setting_value=$(eval generate_${setting} ${cidr})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
            exit 1
          fi
        fi
      else
        echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
      fi
    done
    echo -e "${blue}INFO: Detecting Network Specific settings for: ${enabled_network}${reset}"
    # detect network specific settings
    if [ -n $(eval echo \${${network}_optional_settings}) ]; then
      eval "network_specific_settings=\${${enabled_network}_optional_settings}"
      for setting in ${network_specific_settings}; do
        eval "setting_value=\${${enabled_network}_${setting}}"
        if [ -z "${setting_value}" ]; then
          setting_value=$(eval find_${setting} ${nic_value})
          if [ -n "$setting_value" ]; then
            eval "${enabled_network}_${setting}=${setting_value}"
            echo -e "${blue}INFO: Auto-detection: ${enabled_network}_${setting}: ${setting_value}${reset}"
          else
            eval "cidr=\${${enabled_network}_cidr}"
            setting_value=$(eval generate_${setting} ${cidr})
            if [ -n "$setting_value" ]; then
              eval "${enabled_network}_${setting}=${setting_value}"
              echo -e "${blue}INFO: Auto-generated: ${enabled_network}_${setting}: ${setting_value}${reset}"
            else
              echo -e "${red}ERROR: Auto-generation failed: ${setting} not found${reset}"
              exit 1
            fi
          fi
        else
          echo -e "${blue}INFO: ${enabled_network}_${setting}: ${setting_value}${reset}"
        fi
      done
    fi
  done
}
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
##parses deploy settings yaml into globals and options array
##params: none
##usage:  parse_deploy_settings
parse_deploy_settings() {
  local global_prefix="deploy_global_params_"
  local options_prefix="deploy_deploy_options_"
  local myvar myvalue
  local settings=$(parse_yaml $DEPLOY_SETTINGS_FILE "deploy_")

  for this_setting in $settings; do
    if contains_prefix $this_setting $global_prefix; then
      myvar=$(parse_setting_var $this_setting $global_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      # Do not override variables set by cmdline
      if [ -z "$(eval echo \$$myvar)" ]; then
        eval "$myvar=\$myvalue"
        echo -e "${blue}Global parameter set: ${myvar}:${myvalue}${reset}"
      else
        echo -e "${blue}Global parameter already set: ${myvar}${reset}"
      fi
    elif contains_prefix $this_setting $options_prefix; then
      myvar=$(parse_setting_var $this_setting $options_prefix)
      if [ -z "$myvar" ]; then
        echo -e "${red}ERROR: while parsing ${DEPLOY_SETTINGS_FILE} for setting: ${this_setting}${reset}"
      fi
      myvalue=$(parse_setting_value $this_setting)
      deploy_options_array[$myvar]=$myvalue
      echo -e "${blue}Deploy option set: ${myvar}:${myvalue}${reset}"
    fi
  done
}
##parses baremetal yaml settings into compatible json
##writes the json to $CONFIG/instackenv_tmp.json
##params: none
##usage: parse_inventory_file
parse_inventory_file() {
  local inventory=$(parse_yaml $INVENTORY_FILE)
  local node_list
  local node_prefix="node"
  local node_count=0
  local node_total
  local inventory_list

  # detect number of nodes
  for entry in $inventory; do
    if echo $entry | grep -Eo "^nodes_node[0-9]+_" > /dev/null; then
      this_node=$(echo $entry | grep -Eo "^nodes_node[0-9]+_")
      if [[ $inventory_list != *"$this_node"* ]]; then
        inventory_list+="$this_node "
      fi
    fi
  done

  inventory_list=$(echo $inventory_list | sed 's/ $//')

  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
266
    ((node_count+=1))
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
  done

  node_total=$node_count

  if [[ "$node_total" -lt 5 && ha_enabled == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide at least 5 nodes for HA baremetal deployment${reset}"
    exit 1
  elif [[ "$node_total" -lt 2 ]]; then
    echo -e "${red}ERROR: You must provide at least 2 nodes for non-HA baremetal deployment${reset}"
    exit 1
  fi

  eval $(parse_yaml $INVENTORY_FILE)

  instack_env_output="
{
 \"nodes\" : [

"
  node_count=0
  for node in $inventory_list; do
Dan Radez's avatar
Dan Radez committed
288
289
    ((node_count+=1))
    node_output="
290
291
292
293
294
295
296
297
298
299
300
        {
          \"pm_password\": \"$(eval echo \${${node}ipmi_pass})\",
          \"pm_type\": \"pxe_ipmitool\",
          \"mac\": [
            \"$(eval echo \${${node}mac_address})\"
          ],
          \"cpu\": \"$(eval echo \${${node}cpus})\",
          \"memory\": \"$(eval echo \${${node}memory})\",
          \"disk\": \"$(eval echo \${${node}disk})\",
          \"arch\": \"$(eval echo \${${node}arch})\",
          \"pm_user\": \"$(eval echo \${${node}ipmi_user})\",
301
302
          \"pm_addr\": \"$(eval echo \${${node}ipmi_ip})\",
          \"capabilities\": \"$(eval echo \${${node}capabilities})\"
303
"
Dan Radez's avatar
Dan Radez committed
304
    instack_env_output+=${node_output}
305
306
307
308
309
310
311
312
313
314
315
316
317
    if [ $node_count -lt $node_total ]; then
      instack_env_output+="        },"
    else
      instack_env_output+="        }"
    fi
  done

  instack_env_output+='
  ]
}
'
  #Copy instackenv.json to undercloud for baremetal
  echo -e "{blue}Parsed instackenv JSON:\n${instack_env_output}${reset}"
Dan Radez's avatar
Dan Radez committed
318
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
319
320
321
322
323
324
cat > instackenv.json << EOF
$instack_env_output
EOF
EOI

}
325
326
327
##verify internet connectivity
#params: none
function verify_internet {
Dan Radez's avatar
Dan Radez committed
328
  if ping -c 2 $ping_site > /dev/null; then
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
    if ping -c 2 www.google.com > /dev/null; then
      echo "${blue}Internet connectivity detected${reset}"
      return 0
    else
      echo "${red}Internet connectivity detected, but DNS lookup failed${reset}"
      return 1
    fi
  else
    echo "${red}No internet connectivity detected${reset}"
    return 1
  fi
}

##download dependencies if missing and configure host
#params: none
function configure_deps {
  if ! verify_internet; then
    echo "${red}Will not download dependencies${reset}"
    internet=false
Dan Radez's avatar
Dan Radez committed
348
349
350
351
352
353
  fi

  # verify ip forwarding
  if sysctl net.ipv4.ip_forward | grep 0; then
    sudo sysctl -w net.ipv4.ip_forward=1
    sudo sh -c "echo 'net.ipv4.ip_forward = 1' >> /etc/sysctl.conf"
354
355
  fi

356
357
358
359
360
361
362
363
  # ensure no dhcp server is running on jumphost
  if ! sudo systemctl status dhcpd | grep dead; then
    echo "${red}WARN: DHCP Server detected on jumphost, disabling...${reset}"
    sudo systemctl stop dhcpd
    sudo systemctl disable dhcpd
  fi

  # ensure networks are configured
364
  systemctl start openvswitch
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399

  # If flat we only use admin network
  if [[ "$net_isolation_enabled" == "FALSE" ]]; then
    virsh_enabled_networks="admin_network"
  # For baremetal we only need to create/attach instack to admin and public
  elif [ "$virtual" == "FALSE" ]; then
    virsh_enabled_networks="admin_network public_network"
  else
    virsh_enabled_neworks=$enabled_network_list
  fi

  for network in ${OPNFV_NETWORK_TYPES}; do
    ovs-vsctl list-br | grep ${NET_MAP[$network]} > /dev/null || ovs-vsctl add-br ${NET_MAP[$network]}
    virsh net-list --all | grep ${NET_MAP[$network]} > /dev/null || virsh net-create $CONFIG/${NET_MAP[$network]}-net.xml
    virsh net-list | grep -E "${NET_MAP[$network]}\s+active" > /dev/null || virsh net-start ${NET_MAP[$network]}
  done

  echo -e "${blue}INFO: Bridges set: ${reset}"
  ovs-vsctl list-br
  echo -e "${blue}INFO: virsh networks set: ${reset}"
  virsh net-list

  if [[ -z "$virtual" || "$virtual" == "FALSE" ]]; then
    # bridge interfaces to correct OVS instances for baremetal deployment
    for network in ${enabled_network_list}; do
      this_interface=$(eval echo \${${network}_bridged_interface})
      # check if this a bridged interface for this network
      if [[ -n "$this_interface" || "$this_interface" != "none" ]]; then
        ovs-vsctl list-ports ${NET_MAP[$network]} | grep ${this_interface} || ovs-vsctl add-port ${NET_MAP[$network]} ${this_interface}
      else
        echo "${red}ERROR: Unable to determine interface to bridge to for enabled network: ${network}${reset}"
        exit 1
      fi
    done
  fi
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422

  # ensure storage pool exists and is started
  virsh pool-list --all | grep default > /dev/null || virsh pool-create $CONFIG/default-pool.xml
  virsh pool-list | grep -Eo "default\s+active" > /dev/null || virsh pool-start default

  if virsh net-list | grep default > /dev/null; then
    num_ints_same_subnet=$(ip addr show | grep "inet 192.168.122" | wc -l)
    if [ "$num_ints_same_subnet" -gt 1 ]; then
      virsh net-destroy default
      ##go edit /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' /etc/libvirt/qemu/networks/default.xml
      sed -i 's/192.168.122/192.168.123/g' instackenv-virt.json
      sleep 5
      virsh net-start default
      virsh net-autostart default
    fi
  fi

  if ! egrep '^flags.*(vmx|svm)' /proc/cpuinfo > /dev/null; then
    echo "${red}virtualization extensions not found, kvm kernel module insertion may fail.\n  \
Are you sure you have enabled vmx in your bios or hypervisor?${reset}"
  fi

Dan Radez's avatar
Dan Radez committed
423
424
  if ! lsmod | grep kvm > /dev/null; then modprobe kvm; fi
  if ! lsmod | grep kvm_intel > /dev/null; then modprobe kvm_intel; fi
425
426
427
428
429
430
431
432
433
434
435
436
437

  if ! lsmod | grep kvm > /dev/null; then
    echo "${red}kvm kernel modules not loaded!${reset}"
    return 1
  fi

  ##sshkeygen for root
  if [ ! -e ~/.ssh/id_rsa.pub ]; then
    ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
  fi

  echo "${blue}All dependencies installed and running${reset}"
}
438
439

##verify vm exists, an has a dhcp lease assigned to it
Dan Radez's avatar
Dan Radez committed
440
##params: none
441
function setup_instack_vm {
442
  if ! virsh list --all | grep instack > /dev/null; then
443
      #virsh vol-create default instack.qcow2.xml
444
      virsh define $CONFIG/instack.xml
445

446
447
448
449
      #Upload instack image
      #virsh vol-create default --file instack.qcow2.xml
      virsh vol-create-as default instack.qcow2 30G --format qcow2

450
451
452
453
454
455
456
457
458
      ### this doesn't work for some reason I was getting hangup events so using cp instead
      #virsh vol-upload --pool default --vol instack.qcow2 --file $CONFIG/stack/instack.qcow2
      #2015-12-05 12:57:20.569+0000: 8755: info : libvirt version: 1.2.8, package: 16.el7_1.5 (CentOS BuildSystem <http://bugs.centos.org>, 2015-11-03-13:56:46, worker1.bsys.centos.org)
      #2015-12-05 12:57:20.569+0000: 8755: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #2015-12-05 12:57:20.569+0000: 8756: warning : virKeepAliveTimerInternal:143 : No response from client 0x7ff1e231e630 after 6 keepalive messages in 35 seconds
      #error: cannot close volume instack.qcow2
      #error: internal error: received hangup / error event on socket
      #error: Reconnected to the hypervisor

459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
      instack_dst=/var/lib/libvirt/images/instack.qcow2
      cp -f $RESOURCES/instack.qcow2 $instack_dst

      # resize instack machine
      echo "Checking if instack needs to be resized..."
      instack_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep device | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
      if [ "$instack_size" -lt 30 ]; then
        qemu-img resize /var/lib/libvirt/images/instack.qcow2 +25G
	LIBGUESTFS_BACKEND=direct virt-resize --expand /dev/sda1 $RESOURCES/instack.qcow2 $instack_dst
	LIBGUESTFS_BACKEND=direct virt-customize -a $instack_dst --run-command 'xfs_growfs -d /dev/sda1 || true'
        new_size=$(LIBGUESTFS_BACKEND=direct virt-filesystems --long -h --all -a $instack_dst |grep filesystem | grep -Eo "[0-9\.]+G" | sed -n 's/\([0-9][0-9]*\).*/\1/p')
        if [ "$new_size" -lt 30 ]; then
          echo "Error resizing instack machine, disk size is ${new_size}"
          exit 1
        else
          echo "instack successfully resized"
        fi
      else
        echo "skipped instack resize, upstream is large enough"
      fi
479
480
481
482

  else
      echo "Found Instack VM, using existing VM"
  fi
483
484
485
486

  # if the VM is not running update the authkeys and start it
  if ! virsh list | grep instack > /dev/null; then
    echo "Injecting ssh key to instack VM"
487
488
    virt-customize -c qemu:///system -d instack --run-command "mkdir /root/.ssh/" \
        --upload ~/.ssh/id_rsa.pub:/root/.ssh/authorized_keys \
489
490
491
492
493
494
        --run-command "chmod 600 /root/.ssh/authorized_keys && restorecon /root/.ssh/authorized_keys" \
        --run-command "cp /root/.ssh/authorized_keys /home/stack/.ssh/" \
        --run-command "chown stack:stack /home/stack/.ssh/authorized_keys && chmod 600 /home/stack/.ssh/authorized_keys"
    virsh start instack
  fi

495
496
497
  sleep 3 # let DHCP happen

  CNT=10
498
499
  echo -n "${blue}Waiting for instack's dhcp address${reset}"
  while ! grep instack /var/lib/libvirt/dnsmasq/default.leases > /dev/null && [ $CNT -gt 0 ]; do
500
501
502
503
504
505
      echo -n "."
      sleep 3
      CNT=CNT-1
  done

  # get the instack VM IP
506
  UNDERCLOUD=$(grep instack /var/lib/libvirt/dnsmasq/default.leases | awk '{print $3}' | head -n 1)
Dan Radez's avatar
Dan Radez committed
507
  if [ -z "$UNDERCLOUD" ]; then
508
509
510
    #if not found then dnsmasq may be using leasefile-ro
    instack_mac=$(virsh domiflist instack | grep default | \
                  grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+")
511
    UNDERCLOUD=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'})
512
513
514
515

    if [ -z "$UNDERCLOUD" ]; then
      echo "\n\nNever got IP for Instack. Can Not Continue."
      exit 1
516
517
    else
      echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
518
    fi
Dan Radez's avatar
Dan Radez committed
519
520
521
  else
     echo -e "${blue}\rInstack VM has IP $UNDERCLOUD${reset}"
  fi
522
523

  CNT=10
524
  echo -en "${blue}\rValidating instack VM connectivity${reset}"
525
526
527
  while ! ping -c 1 $UNDERCLOUD > /dev/null && [ $CNT -gt 0 ]; do
      echo -n "."
      sleep 3
528
      CNT=$CNT-1
529
  done
Dan Radez's avatar
Dan Radez committed
530
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
531
532
533
      echo "Failed to contact Instack. Can Not Continue"
      exit 1
  fi
534
  CNT=10
535
  while ! ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "echo ''" 2>&1> /dev/null && [ $CNT -gt 0 ]; do
536
537
      echo -n "."
      sleep 3
538
      CNT=$CNT-1
539
  done
Dan Radez's avatar
Dan Radez committed
540
  if [ "$CNT" -eq 0 ]; then
Dan Radez's avatar
Dan Radez committed
541
542
543
      echo "Failed to connect to Instack. Can Not Continue"
      exit 1
  fi
544
545

  # extra space to overwrite the previous connectivity output
Dan Radez's avatar
Dan Radez committed
546
  echo -e "${blue}\r                                                                 ${reset}"
547

548
549
550
551
552
553
  #add the instack public interface if net isolation is enabled (more than just admin network)
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    virsh attach-interface --domain instack --type network --source ${NET_MAP['public_network']} --model rtl8139 --config --live
    sleep 1
    ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "if ! ip a s eth2 | grep ${public_network_provisioner_ip} > /dev/null; then ip a a ${public_network_provisioner_ip}/${public_network_cidr##*/} dev eth2; ip link set up dev eth2; fi"
  fi
554
555
556
  # ssh key fix for stack user
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "restorecon -r /home/stack"
}
557

Tim Rozet's avatar
Tim Rozet committed
558
559
##Create virtual nodes in virsh
##params: none
560
561
function setup_virtual_baremetal {
  for i in $(seq 0 $vm_index); do
562
563
564
    if ! virsh list --all | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} > /dev/null; then
      if [ ! -e $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml ]; then
        define_virtual_node baremetalbrbm_brbm1_brbm2_brbm3_${i}
Tim Rozet's avatar
Tim Rozet committed
565
      fi
566
567
568
569
570
571
572
      # Fix for ramdisk using wrong pxeboot interface
      # TODO: revisit this and see if there's a more proper fix
      sed -i "/^\s*<source network='brbm2'\/>/{
        N
        s/^\(.*\)virtio\(.*\)$/\1rtl8139\2/
        }" $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
      virsh define $CONFIG/baremetalbrbm_brbm1_brbm2_brbm3_${i}.xml
573
574
575
    else
      echo "Found Baremetal ${i} VM, using existing VM"
    fi
576
    virsh vol-list default | grep baremetalbrbm_brbm1_brbm2_brbm3_${i} 2>&1> /dev/null || virsh vol-create-as default baremetalbrbm_brbm1_brbm2_brbm3_${i}.qcow2 40G --format qcow2
577
  done
578

579
580
}

581
582
583
584
585
586
587
588
589
##Set network-environment settings
##params: network-environment file to edit
function configure_network_environment {
  sed -i '/ControlPlaneSubnetCidr/c\\  ControlPlaneSubnetCidr: "'${admin_network_cidr##*/}'"' $1
  sed -i '/ControlPlaneDefaultRoute/c\\  ControlPlaneDefaultRoute: '${admin_network_provisioner_ip}'' $1
  sed -i '/ExternalNetCidr/c\\  ExternalNetCidr: '${public_network_cidr}'' $1
  sed -i "/ExternalAllocationPools/c\\  ExternalAllocationPools: [{'start': '${public_network_usable_ip_range%%,*}', 'end': '${public_network_usable_ip_range##*,}'}]" $1
  sed -i '/ExternalInterfaceDefaultRoute/c\\  ExternalInterfaceDefaultRoute: '${public_network_gateway}'' $1
}
590
##Copy over the glance images and instack json file
Tim Rozet's avatar
Tim Rozet committed
591
##params: none
592
function configure_undercloud {
593

594
595
  echo
  echo "Copying configuration file and disk images to instack"
596
  scp ${SSH_OPTIONS[@]} $RESOURCES/overcloud-full.qcow2 "stack@$UNDERCLOUD":
597
598
599
600
601
602
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    configure_network_environment $CONFIG/network-environment.yaml
    echo -e "${blue}Network Environment set for Deployment: ${reset}"
    cat $CONFIG/network-environment.yaml
    scp ${SSH_OPTIONS[@]} $CONFIG/network-environment.yaml "stack@$UNDERCLOUD":
  fi
Dan Radez's avatar
Dan Radez committed
603
  scp ${SSH_OPTIONS[@]} -r $CONFIG/nics/ "stack@$UNDERCLOUD":
604

605
606
607
  # ensure stack user on instack machine has an ssh key
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "if [ ! -e ~/.ssh/id_rsa.pub ]; then ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa; fi"

608
  if [ "$virtual" == "TRUE" ]; then
609
610
611
612
613
614

      # copy the instack vm's stack user's pub key to
      # root's auth keys so that instack can control
      # vm power on the hypervisor
      ssh ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys

615
616
617
618
619
620
621
      # fix MACs to match new setup
      for i in $(seq 0 $vm_index); do
        pyscript="import json
data = json.load(open('$CONFIG/instackenv-virt.json'))
print data['nodes'][$i]['mac'][0]"

        old_mac=$(python -c "$pyscript")
622
        new_mac=$(virsh dumpxml baremetalbrbm_brbm1_brbm2_brbm3_$i | grep "mac address" | cut -d = -f2 | grep -Eo "[0-9a-f:]+")
623
        # this doesn't work with multiple vnics on the vms
Dan Radez's avatar
Dan Radez committed
624
625
626
627
        #if [ "$old_mac" != "$new_mac" ]; then
        #  echo "${blue}Modifying MAC for node from $old_mac to ${new_mac}${reset}"
        #  sed -i 's/'"$old_mac"'/'"$new_mac"'/' $CONFIG/instackenv-virt.json
        #fi
628
629
      done

630
      DEPLOY_OPTIONS+=" --libvirt-type qemu"
631
      INSTACKENV=$CONFIG/instackenv-virt.json
632

633
634
635
      # upload instackenv file to Instack for virtual deployment
      scp ${SSH_OPTIONS[@]} $INSTACKENV "stack@$UNDERCLOUD":instackenv.json
  fi
636
637
638

  # allow stack to control power management on the hypervisor via sshkey
  # only if this is a virtual deployment
639
  if [ "$virtual" == "TRUE" ]; then
640
641
642
643
644
645
646
647
648
      ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
while read -r line; do
  stack_key=\${stack_key}\\\\\\\\n\${line}
done < <(cat ~/.ssh/id_rsa)
stack_key=\$(echo \$stack_key | sed 's/\\\\\\\\n//')
sed -i 's~INSERT_STACK_USER_PRIV_KEY~'"\$stack_key"'~' instackenv.json
EOI
  fi

649
650
651
  # copy stack's ssh key to this users authorized keys
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> ~/.ssh/authorized_keys

652
653
654
  # disable requiretty for sudo
  ssh -T ${SSH_OPTIONS[@]} "root@$UNDERCLOUD" "sed -i 's/Defaults\s*requiretty//'" /etc/sudoers

655
656
657
658
  # configure undercloud on Undercloud VM
  echo "Running undercloud configuration."
  echo "Logging undercloud configuration to instack:/home/stack/apex-undercloud-install.log"
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" << EOI
659
if [[ "$net_isolation_enabled" == "TRUE" ]]; then
660
661
662
663
664
665
666
667
  sed -i 's/#local_ip/local_ip/' undercloud.conf
  sed -i 's/#network_gateway/network_gateway/' undercloud.conf
  sed -i 's/#network_cidr/network_cidr/' undercloud.conf
  sed -i 's/#dhcp_start/dhcp_start/' undercloud.conf
  sed -i 's/#dhcp_end/dhcp_end/' undercloud.conf
  sed -i 's/#inspection_iprange/inspection_iprange/' undercloud.conf
  sed -i 's/#undercloud_debug/undercloud_debug/' undercloud.conf

668
669
670
671
672
673
  openstack-config --set undercloud.conf DEFAULT local_ip ${admin_network_provisioner_ip}/${admin_network_cidr##*/}
  openstack-config --set undercloud.conf DEFAULT network_gateway ${admin_network_provisioner_ip}
  openstack-config --set undercloud.conf DEFAULT network_cidr ${admin_network_cidr}
  openstack-config --set undercloud.conf DEFAULT dhcp_start ${admin_network_dhcp_range%%,*}
  openstack-config --set undercloud.conf DEFAULT dhcp_end ${admin_network_dhcp_range##*,}
  openstack-config --set undercloud.conf DEFAULT inspection_iprange ${admin_network_introspection_range}
674
675
676
677
  openstack-config --set undercloud.conf DEFAULT undercloud_debug false
fi

openstack undercloud install &> apex-undercloud-install.log
678
679
680
681
sleep 30
sudo systemctl restart openstack-glance-api
sudo systemctl restart openstack-nova-conductor
sudo systemctl restart openstack-nova-compute
682
EOI
683
684
685
686
687
688
689
# WORKAROUND: must restart the above services to fix sync problem with nova compute manager
# TODO: revisit and file a bug if necessary. This should eventually be removed
# as well as glance api problem
echo -e "${blue}INFO: Sleeping 15 seconds while services come back from restart${reset}"
sleep 15
#TODO Fill in the rest of the network-environment values for other networks

690
691
692
}

##preping it for deployment and launch the deploy
Tim Rozet's avatar
Tim Rozet committed
693
##params: none
694
function undercloud_prep_overcloud_deploy {
695
696
697
698
699
700
701
702
703
704
705
706

  if [[ ${#deploy_options_array[@]} -eq 0 || ${deploy_options_array['sdn_controller']} == 'opendaylight' ]]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'opendaylight-external' ]; then
    DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/opendaylight-external.yaml"
  elif [ ${deploy_options_array['sdn_controller']} == 'onos' ]; then
    echo -e "${red}ERROR: ONOS is currently unsupported...exiting${reset}"
    exit 1
  elif [ ${deploy_options_array['sdn_controller']} == 'opencontrail' ]; then
    echo -e "${red}ERROR: OpenContrail is currently unsupported...exiting${reset}"
    exit 1
  fi
707

Tim Rozet's avatar
Tim Rozet committed
708
  # check if HA is enabled
709
  if [[ "$ha_enabled" == "TRUE" ]]; then
Tim Rozet's avatar
Tim Rozet committed
710
711
712
     DEPLOY_OPTIONS+=" --control-scale 3 --compute-scale 2"
     DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/puppet-pacemaker.yaml"
  fi
713

714
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
715
     #DEPLOY_OPTIONS+=" -e /usr/share/openstack-tripleo-heat-templates/environments/network-isolation.yaml"
716
     DEPLOY_OPTIONS+=" -e network-environment.yaml"
717
718
  fi

719
  if [[ "$ha_enabled" == "TRUE" ]] || [[ "$net_isolation_enabled" == "TRUE" ]]; then
Dan Radez's avatar
Dan Radez committed
720
     DEPLOY_OPTIONS+=" --ntp-server $ntp_server"
721
722
  fi

723
724
725
726
  if [[ ! "$virtual" == "TRUE" ]]; then
     DEPLOY_OPTIONS+=" --control-flavor control --compute-flavor compute"
  fi

727
  ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
728
source stackrc
729
730
set -o errexit
echo "Uploading overcloud glance images"
731
openstack overcloud image upload
732
echo "Configuring undercloud and discovering nodes"
733
734
735
openstack baremetal import --json instackenv.json
openstack baremetal configure boot
openstack baremetal introspection bulk start
736
echo "Configuring flavors"
737
738
739
740
741
742
743
744
745
746
for flavor in baremetal control compute; do
  echo -e "${blue}INFO: Updating flavor: \${flavor}${reset}"
  if openstack flavor list | grep \${flavor}; then
    openstack flavor delete \${flavor}
  fi
  openstack flavor create --id auto --ram 4096 --disk 39 --vcpus 1 \${flavor}
  if ! openstack flavor list | grep \${flavor}; then
    echo -e "${red}ERROR: Unable to create flavor \${flavor}${reset}"
  fi
done
747
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" baremetal
748
749
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="control" control
openstack flavor set --property "cpu_arch"="x86_64" --property "capabilities:boot_option"="local" --property "capabilities:profile"="compute" compute
750
751
echo "Configuring nameserver on ctlplane network"
neutron subnet-update \$(neutron subnet-list | grep -v id | grep -v \\\\-\\\\- | awk {'print \$2'}) --dns-nameserver 8.8.8.8
752
echo "Executing overcloud deployment, this should run for an extended period without output."
753
sleep 60 #wait for Hypervisor stats to check-in to nova
754
openstack overcloud deploy --templates $DEPLOY_OPTIONS
755
756
757
758
759
EOI

}

display_usage() {
Dan Radez's avatar
Dan Radez committed
760
761
  echo -e "Usage:\n$0 [arguments] \n"
  echo -e "   -c|--config : Directory to configuration files. Optional.  Defaults to /var/opt/opnfv/ \n"
762
763
  echo -e "   -d|--deploy-settings : Full path to deploy settings yaml file. Optional.  Defaults to null \n"
  echo -e "   -i|--inventory : Full path to inventory yaml file. Required only for baremetal \n"
764
  echo -e "   -n|--net-settings : Full path to network settings file. Optional. \n"
Dan Radez's avatar
Dan Radez committed
765
766
767
  echo -e "   -p|--ping-site : site to use to verify IP connectivity. Optional. Defaults to 8.8.8.8 \n"
  echo -e "   -r|--resources : Directory to deployment resources. Optional.  Defaults to /var/opt/opnfv/stack \n"
  echo -e "   -v|--virtual : Virtualize overcloud nodes instead of using baremetal. \n"
768
  echo -e "   --no-ha : disable High Availability deployment scheme, this assumes a single controller and single compute node \n"
769
  echo -e "   --flat : disable Network Isolation and use a single flat network for the underlay network."
770
771
}

772
##translates the command line parameters into variables
773
774
775
776
777
778
779
##params: $@ the entire command line is passed
##usage: parse_cmd_line() "$@"
parse_cmdline() {
  echo -e "\n\n${blue}This script is used to deploy the Apex Installer and Provision OPNFV Target System${reset}\n\n"
  echo "Use -h to display help"
  sleep 2

Dan Radez's avatar
Dan Radez committed
780
  while [ "${1:0:1}" = "-" ]
781
782
  do
    case "$1" in
783
784
785
786
787
788
        -h|--help)
                display_usage
                exit 0
            ;;
        -c|--config)
                CONFIG=$2
789
                echo "Deployment Configuration Directory Overridden to: $2"
790
791
                shift 2
            ;;
792
793
794
795
796
797
798
        -d|--deploy-settings)
                DEPLOY_SETTINGS_FILE=$2
                echo "Deployment Configuration file: $2"
                shift 2
            ;;
        -i|--inventory)
                INVENTORY_FILE=$2
799
800
                shift 2
            ;;
801
802
803
        -n|--net-settings)
                NETSETS=$2
                echo "Network Settings Configuration file: $2"
804
805
806
807
                shift 2
            ;;
        -p|--ping-site)
                ping_site=$2
808
                echo "Using $2 as the ping site"
809
810
                shift 2
            ;;
811
812
        -r|--resources)
                RESOURCES=$2
813
                echo "Deployment Resources Directory Overridden to: $2"
814
                shift 2
815
            ;;
816
        -v|--virtual)
817
                virtual="TRUE"
818
                echo "Executing a Virtual Deployment"
819
820
                shift 1
            ;;
821
        --no-ha )
822
823
                ha_enabled="FALSE"
                echo "HA Deployment Disabled"
Tim Rozet's avatar
Tim Rozet committed
824
                shift 1
825
            ;;
826
        --flat )
827
828
                net_isolation_enabled="FALSE"
                echo "Underlay Network Isolation Disabled: using flat configuration"
829
830
                shift 1
            ;;
831
832
833
834
835
836
        *)
                display_usage
                exit 1
            ;;
    esac
  done
837

838
839
840
841
  if [[ ! -z "$NETSETS" && "$net_isolation_enabled" == "FALSE" ]]; then
    echo -e "${red}INFO: Single flat network requested. Ignoring any network settings!${reset}"
  elif [[ -z "$NETSETS" && "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${red}ERROR: You must provide a network_settings file with -n or use --flat to force a single flat network{reset}"
842
843
844
845
846
847
848
849
850
851
852
853
  fi

  if [[ -n "$virtual" && -n "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You should not specify an inventory with virtual deployments${reset}"
    exit 1
  fi

  if [[ ! -z "$DEPLOY_SETTINGS_FILE" && ! -f "$DEPLOY_SETTINGS_FILE" ]]; then
    echo -e "${red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

854
855
  if [[ ! -z "$NETSETS" && ! -f "$NETSETS" ]]; then
    echo -e "${red}ERROR: ${NETSETS} does not exist! Exiting...${reset}"
856
857
858
859
860
861
862
863
864
865
866
867
    exit 1
  fi

  if [[ ! -z "$INVENTORY_FILE" && ! -f "$INVENTORY_FILE" ]]; then
    echo -e "{$red}ERROR: ${DEPLOY_SETTINGS_FILE} does not exist! Exiting...${reset}"
    exit 1
  fi

  if [[ -z "$virtual" && -z "$INVENTORY_FILE" ]]; then
    echo -e "${red}ERROR: You must specify an inventory file for baremetal deployments! Exiting...${reset}"
    exit 1
  fi
868
869
870
871
872
873
}

##END FUNCTIONS

main() {
  parse_cmdline "$@"
874
875
876
877
  if [[ "$net_isolation_enabled" == "TRUE" ]]; then
    echo -e "${blue}INFO: Parsing network settings file...${reset}"
    parse_network_settings
  fi
Dan Radez's avatar
Dan Radez committed
878
  if ! configure_deps; then
879
880
    echo -e "${red}Dependency Validation Failed, Exiting.${reset}"
    exit 1
Dan Radez's avatar
Dan Radez committed
881
  fi
882
883
884
  if [ -n "$DEPLOY_SETTINGS_FILE" ]; then
    parse_deploy_settings
  fi
885
  setup_instack_vm
886
  if [ "$virtual" == "TRUE" ]; then
887
    setup_virtual_baremetal
888
889
  elif [ -n "$INVENTORY_FILE" ]; then
    parse_inventory_file
890
  fi
891
  configure_undercloud
892
893
894
895
  undercloud_prep_overcloud_deploy
}

main "$@"