Commit 02831889 authored by Dan Radez's avatar Dan Radez
Browse files

Adding initial build scripts based on RDO manager



Change-Id: Ib5f8ed4dfcaa42bcefcf2a7914c201a887beeacd
Signed-off-by: default avatarDan Radez <dradez@redhat.com>
parent a5f42c9d
##############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# stefan.k.berg@ericsson.com
# jonas.bjurel@ericsson.com
# dradez@redhat.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
SHELL = /bin/bash
############################################################################
# BEGIN of variables to customize
#
#Input args
export UNIT_TEST = FALSE
export INTERACTIVE = TRUE
export CENTDNLD = http://mirrors.cat.pdx.edu/centos/7.1.1503/isos/x86_64/CentOS-7-x86_64-DVD-1503-01.iso
export ISOSRC = file:$(shell pwd)/CentOS-7-x86_64-DVD-1503-01.iso
export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
export PRODNO = "OPNFV_BGS"
export REVSTATE = "P0000"
export NEWISO = $(shell pwd)/release/OPNFV-CentOS-7-x86_64-${REVSTATE}.iso
export GENESISRPM = $(shell pwd)/x86_64/opnfv-genesis-0.1-1.x86_64.rpm
# Note! Invoke with "make REVSTATE=RXXXX all" to make release build!
# Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location.
#Build variables
export BUILD_BASE := $(shell pwd)
export CACHE_DIR := $(BUILD_BASE)/cache
export VERSION_FILE := $(BUILD_BASE)/.versions
export TOPDIR := $(shell pwd)
CENTDIR := $(TOPDIR)/centiso
#
# END of variables to customize
#############################################################################
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
.PHONY: all
all: iso
@echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
@echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
@echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
############################################################################
# BEGIN of Include definitions
#
include config.mk
include cache.mk
#
# END Include definitions
#############################################################################
$(ISOCACHE):
test -s $(ISOCACHE) || { wget -nv $(CENTDNLD) ; }
.PHONY: mount-centiso umount-centiso
mount-centiso: $(ISOCACHE)
@echo "Mounting CentOS ISO in $(CENTDIR)"
@mkdir -p $(CENTDIR)
@fuseiso $(ISOCACHE) $(CENTDIR)
umount-centiso:
@set +e
@echo "Unmounting CentOS ISO from $(CENTDIR)"
@fusermount -u $(CENTDIR)
@rmdir $(CENTDIR)
@set -e
.PHONY: build-clean $(SUBCLEAN)
build-clean: $(SUBCLEAN)
@rm -Rf centos
@rm -Rf release
@rm -Rf newiso
@rm -f $(NEWISO)
.PHONY: clean $(SUBCLEAN)
clean: clean-cache $(SUBCLEAN)
@rm -f *.iso
@rm -Rf release
@rm -Rf newiso
@rm -f $(NEWISO)
@rm -f $(BUILD_BASE)/.versions
$(SUBCLEAN): %.clean:
$(MAKE) -C $* -f Makefile clean
.PHONY: rpm-clean
rpm-clean:
rpmbuild --clean opnfv-genesis.spec -D '_topdir %(echo `pwd`)' -D '_builddir %(echo `pwd`)' -D '_sourcedir %(echo `pwd`)' -D '_rpmdir %(echo `pwd`)' -D '_specdir %(echo `pwd`)' -D '_srcrpmdir %(echo `pwd`)'
.PHONY: rpm
rpm:
pushd ../../ && git archive --format=tar --prefix=opnfv-genesis-0.1/ HEAD | gzip > foreman/build/opnfv-genesis.tar.gz
rpmbuild -ba opnfv-genesis.spec -D '_topdir %(echo `pwd`)' -D '_builddir %(echo `pwd`)' -D '_sourcedir %(echo `pwd`)' -D '_rpmdir %(echo `pwd`)' -D '_specdir %(echo `pwd`)' -D '_srcrpmdir %(echo `pwd`)'
@make rpm-clean
.PHONY: instack
instack: instack.qcow2
instack.qcow2:
@./instack.sh
.PHONY: instack-clean
instack-clean:
@virsh destroy instack 2> /dev/null || echo -n ''
@virsh undefine instack 2> /dev/null || echo -n ''
@virsh destroy baremetal_0 2> /dev/null || echo -n ''
@virsh undefine baremetal_0 2> /dev/null || echo -n ''
@virsh destroy baremetal_1 2> /dev/null || echo -n ''
@virsh undefine baremetal_1 2> /dev/null || echo -n ''
rm -f instack.xml
rm -f instack.qcow2
rm -rf stack
# Todo: Make things smarter - we shouldn't need to clean everything
# betwen make invocations.
.PHONY: iso
iso: build-clean $(ISOCACHE) instack.qcow2 rpm
@make mount-centiso
@mkdir centos release
cp -r $(CENTDIR)/* centos
@make umount-centiso
# modify the installer iso's contents
@cp -f isolinux.cfg centos/isolinux/isolinux.cfg
@cp $(GENESISRPM) centos/Packages
# regenerate yum repo data
@echo "Generating new yum metadata"
createrepo --update -g ../c7-opnfv-x86_64-comps.xml centos
# build the iso
@echo "Building OPNFV iso"
mkisofs -b isolinux/isolinux.bin -no-emul-boot -boot-load-size 4 -boot-info-table -V "OPNFV CentOS 7 x86_64" -R -J -v -T -o $(NEWISO) centos
@printf "\n\nISO is built at $(NEWISO)\n\n"
##############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# stefan.k.berg@ericsson.com
# jonas.bjurel@ericsson.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
SHELL = /bin/bash
CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS))
CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS))
############################################################################
# BEGIN of variables to customize
#
CACHEFILES += .versions
CACHEFILES += $(shell basename $(ISOSRC))
#
# END of variables to customize
############################################################################
.PHONY: prepare-cache
prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES)
.PHONY: make-cache-dir
make-cache-dir:
@rm -rf ${CACHE_DIR}
@mkdir ${CACHE_DIR}
.PHONY: clean-cache
clean-cache: $(CACHECLEAN)
@rm -rf ${CACHE_DIR}
.PHONY: $(CACHEDIRS)
$(CACHEDIRS):
@mkdir -p $(dir $(CACHE_DIR)/$@)
@if [ ! -d $(BUILD_BASE)/$@ ]; then\
mkdir -p $(BUILD_BASE)/$@;\
fi
@ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@
.PHONY: $(CACHEFILES)
$(CACHEFILES):
@mkdir -p $(dir $(CACHE_DIR)/$@)
@if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\
mkdir -p $(dir $(BUILD_BASE)/$@);\
fi
@if [ ! -f $(BUILD_BASE)/$@ ]; then\
echo " " > $(BUILD_BASE)/$@;\
ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
rm -f $(BUILD_BASE)/$@;\
else\
ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
fi
.PHONY: validate-cache
validate-cache: $(CACHEVALIDATE)
@if [[ $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}') ]]; then\
echo "Cache does not match current config.mk definition, cache must be rebuilt";\
exit 1;\
fi;
@if [[ $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}') ]]; then\
echo "Cache does not match current cache.mk definition, cache must be rebuilt";\
exit 1;\
fi;
.PHONY: $(CACHEVALIDATE)
$(CACHEVALIDATE): %.validate:
@echo VALIDATE $(CACHEVALIDATE)
$(MAKE) -C $* -f Makefile validate-cache
.PHONY: $(CACHECLEAN)
$(CACHECLEAN): %.clean:
rm -rf ${CACHE_DIR}/$*
#!/bin/sh
set -e
if ! id stack > /dev/null; then
useradd stack;
echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack
echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack
chmod 0440 /etc/sudoers.d/stack
echo 'Added user stack'
fi
if ! rpm -q epel-release > /dev/null; then
yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
fi
if ! rpm -q rdo-release > /dev/null; then
yum install -y https://rdoproject.org/repos/openstack-kilo/rdo-release-kilo.rpm
fi
if ! [ -a /etc/yum.repos.d/rdo-management-trunk.repo ]; then
curl -o /etc/yum.repos.d/rdo-management-trunk.repo http://trunk-mgt.rdoproject.org/centos-kilo/current-passed-ci/delorean-rdo-management.repo
fi
if ! rpm -q instack-undercloud > /dev/null; then
yum install -y instack-undercloud
fi
sudo -u stack -- sh -c 'cd; instack-virt-setup'
#if ! ovs-vsctl show | grep brbm; then
# ovs-vsctl add-port brbm em2
#fi
cp /home/stack/.ssh/id_rsa* /root/.ssh/
UNDERCLOUD=$(virsh net-dhcp-leases default | grep instack | awk '{print $5}' | awk -F '/' '{print $1}')
ssh -T -o "StrictHostKeyChecking no" root@$UNDERCLOUD <<EOI
if ! rpm -q rdo-release > /dev/null; then
yum install -y https://rdoproject.org/repos/openstack-kilo/rdo-release-kilo.rpm
fi
if ! [ -a /etc/yum.repos.d/rdo-management-trunk.repo ]; then
curl -o /etc/yum.repos.d/rdo-management-trunk.repo http://trunk-mgt.rdoproject.org/centos-kilo/current-passed-ci/delorean-rdo-management.repo
fi
yum install -y python-rdomanager-oscplugin
cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys
chown stack:stack /home/stack/.ssh/authorized_keys
EOI
ssh -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "openstack undercloud install"
#ssh -T -o "StrictHostKeyChecking no" stack@$UNDERCLOUD <<EOI
#echo "Running undercloud install"
#openstack undercloud install
#dontexit=\$(openstack undercloud install)
#openstack undercloud install && sudo halt -p
#EOI
echo "Shuttind down instack to take snapshop"
virsh shutdown instack
echo "Waiting for instack VM to shutdown"
while virsh list | grep instack; do
echo -n "."
sleep 5
done
echo "Copying instack disk image and starting instack VM."
cp -f /var/lib/libvirt/images/instack.qcow2 .
virsh dumpxml instack > instack.xml
virsh vol-dumpxml instack.qcow2 --pool default > instack.qcow2.xml
virsh start instack
echo "Waiting for instack VM to start"
while ! ping -c 1 $UNDERCLOUD > /dev/null; do
echo -n "."
sleep 5
done
while ! ssh -T -o "StrictHostKeyChecking no" root@$UNDERCLOUD "echo ''" > /dev/null; do
echo -n "."
sleep 3
done
echo "Copying CentOS Cache to instack VM"
ssh -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "mkdir .cache"
scp -r /home/stack/.cache/image-create/CentOS-7-x86_64-GenericCloud* stack@$UNDERCLOUD:.cache/
echo "Building overcloud images"
ssh -tt -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "openstack overcloud image build --all"
echo "Copying overcloud images"
mkdir stack
scp stack@$UNDERCLOUD:deploy-ramdisk-ironic.initramfs stack
scp stack@$UNDERCLOUD:deploy-ramdisk-ironic.kernel stack
scp stack@$UNDERCLOUD:discovery-ramdisk.initramfs stack
scp stack@$UNDERCLOUD:discovery-ramdisk.kernel stack
scp stack@$UNDERCLOUD:fedora-user.qcow2 stack
scp stack@$UNDERCLOUD:overcloud-full.initrd stack
scp stack@$UNDERCLOUD:overcloud-full.qcow2 stack
scp stack@$UNDERCLOUD:overcloud-full.vmlinuz stack
#!/bin/bash
set -e
##############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# stefan.k.berg@ericsson.com
# jonas.bjurel@ericsson.com
# dradez@redhat.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
trap 'echo "Exiting ..."; \
if [ -f ${LOCK_FILE} ]; then \
if [ $(cat ${LOCK_FILE}) -eq $$ ]; then \
rm -f ${LOCK_FILE}; \
fi; \
fi;' EXIT
############################################################################
# BEGIN of usage description
#
usage ()
{
cat << EOF
$0 Builds the Foreman OPNFV Deployment ISO
usage: $0 [-s spec-file] [-c cache-URI] [-l log-file] [-f Flags] build-directory
OPTIONS:
-s spec-file ($BUILD_SPEC), define the build-spec file, default ../build/config.mk
-c cache base URI ($BUILD_CACHE_URI), specifies the base URI to a build cache to be used/updated - the name is automatically generated from the md5sum of the spec-file, http://, ftp://, file://[absolute path] suported.
-l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal
-v version tag to be applied to the build result
-r alternative remote access method script/program. curl is default.
-t run small build-script unit test.
-T run large build-script unit test.
-f build flags ($BUILD_FLAGS):
o s: Do nothing, succeed
o f: Do nothing, fail
o t: run build unit tests
o i: run interactive (-t flag to docker run)
o P: Populate a new local cache and push it to the (-c cache-URI) cache artifactory if -c option is present, currently file://, http:// and ftp:// are supported
o d: Detatch - NOT YET SUPPORTED
build-directory ($BUILD_DIR), specifies the directory for the output artifacts (.iso file).
-h help, prints this help text
Description:
build.sh builds opnfv .iso artifact.
To reduce build time it uses build cache on a local or remote location. The cache is rebuilt and uploaded if either of the below conditions are met:
1) The P(opulate) flag is set and the -c cache-base-URI is provided, if -c is not provided the cache will stay local.
2) If the cache is invalidated by one of the following conditions:
- The config spec md5sum does not compare to the md5sum for the spec which the cache was built.
- The git Commit-Id on the remote repos/HEAD defined in the spec file does not correspont with the Commit-Id for what the cache was built with.
3) A valid cache does not exist on the specified -c cache-base-URI.
The cache URI object name is foreman_cache-"md5sum(spec file)"
Logging by default to console, but can be directed elsewhere with the -l option in which case both stdout and stderr is redirected to that destination.
Built in unit testing of components is enabled by adding the t(est) flag.
Return codes:
- 0 Success!
- 1-99 Unspecified build error
- 100-199 Build system internal error (not build it self)
o 101 Build system instance busy
- 200 Build failure
Examples:
build -c http://opnfv.org/artifactory/foreman/cache -d ~/jenkins/genesis/foreman/ci/output -f ti
NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work
EOF
}
#
# END of usage description
############################################################################
############################################################################
# BEGIN of variables to customize
#
BUILD_BASE=$(readlink -e ../build/)
RESULT_DIR="${BUILD_BASE}/release"
BUILD_SPEC="${BUILD_BASE}/config.mk"
CACHE_DIR="cache"
LOCAL_CACHE_ARCH_NAME="foreman-cache"
REMOTE_CACHE_ARCH_NAME="foreman_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")"
REMOTE_ACCESS_METHD=curl
INCLUDE_DIR=../include
#
# END of variables to customize
############################################################################
############################################################################
# BEGIN of script assigned variables
#
SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
LOCK_FILE="${SCRIPT_DIR}/.build.lck"
CACHE_TMP="${SCRIPT_DIR}/tmp"
TEST_SUCCEED=0
TEST_FAIL=0
UNIT_TEST=0
UPDATE_CACHE=0
POPULATE_CACHE=0
RECURSIV=0
DETACH=0
DEBUG=0
INTEGRATION_TEST=0
FULL_INTEGRATION_TEST=0
INTERACTIVE=0
BUILD_CACHE_URI=
BUILD_SPEC=
BUILD_DIR=
BUILD_LOG=
BUILD_VERSION=
MAKE_ARGS=
#
# END of script assigned variables
############################################################################
############################################################################
# BEGIN of include pragmas
#
source ${INCLUDE_DIR}/build.sh.debug
#
# END of include
############################################################################
############################################################################
# BEGIN of main
#
while getopts "s:c:d:v:f:l:r:RtTh" OPTION
do
case $OPTION in
h)
usage
rc=0
exit $rc
;;
s)
BUILD_SPEC=${OPTARG}
;;
c)
BUILD_CACHE_URI=${OPTARG}
;;
d)
BUILD_DIR=${OPTARG}
;;
l)
BUILD_LOG=${OPTARG}
;;
v)
BUILD_VERSION=${OPTARG}
;;
f)
BUILD_FLAGS=${OPTARG}
;;
r) REMOTE_ACCESS_METHD=${OPTARG}
;;
R)
RECURSIVE=1
;;
t)
INTEGRATION_TEST=1
;;
T)
INTEGRATION_TEST=1
FULL_INTEGRATION_TEST=1
;;
*)
echo "${OPTION} is not a valid argument"
rc=100
exit $rc
;;
esac
done
if [ -z $BUILD_DIR ]; then
BUILD_DIR=$(echo $@ | cut -d ' ' -f ${OPTIND})
fi
for ((i=0; i<${#BUILD_FLAGS};i++)); do
case ${BUILD_FLAGS:$i:1} in
s)
rc=0
exit $rc
;;
f)
rc=1
exit $rc
;;
t)
UNIT_TEST=1
;;
i)
INTERACTIVE=1
;;
P)
POPULATE_CACHE=1
;;
d)
DETACH=1
echo "Detach is not yet supported - exiting ...."
rc=100
exit $rc
;;
D)
DEBUG=1
;;
*)
echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...."
rc=100
exit $rc
;;
esac
done
shift $((OPTIND-1))
if [ ${INTEGRATION_TEST} -eq 1 ]; then
integration-test
rc=0
exit $rc
fi
if [ ! -f ${BUILD_SPEC} ]; then
echo "spec file does not exist: $BUILD_SPEC - exiting ...."
rc=100
exit $rc
fi
if [ -z ${BUILD_DIR} ]; then
echo "Missing build directory - exiting ...."
rc=100
exit $rc
fi
if [ ! -z ${BUILD_LOG} ]; then
if [[ ${RECURSIVE} -ne 1 ]]; then
set +e
eval $0 -R $@ > ${BUILD_LOG} 2>&1
rc=$?
set -e
if [ $rc -ne 0]; then
exit $rc
fi
fi
fi
if [ ${TEST_SUCCEED} -eq 1 ]; then
sleep 1
rc=0
exit $rc
fi
if [ ${TEST_FAIL} -eq 1 ]; then
sleep 1
rc=1
exit $rc
fi
if [ -e ${LOCK_FILE} ]; then
echo "A build job is already running, exiting....."
rc=101
exit $rc
fi
echo $$ > ${LOCK_FILE}
if [ ! -z ${BUILD_CACHE_URI} ]; then
if [ ${POPULATE_CACHE} -ne 1 ]; then
rm -rf ${CACHE_TMP}/cache
mkdir -p ${CACHE_TMP}/cache
echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..."
set +e
${REMOTE_ACCESS_METHD} -o ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
rc=$?
set -e
if [ $rc -ne 0 ]; then
echo "Remote cache does not exist, or is not accessible - a new cache will be built ..."
POPULATE_CACHE=1
else
echo "Unpacking cache file ..."
tar -C ${CACHE_TMP}/cache -xvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz
cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/.
set +e
make -C ${BUILD_BASE} validate-cache;
rc=$?
set -e
if [ $rc -ne 0 ]; then
echo "Cache invalid - a new cache will be built "
POPULATE_CACHE=1
else
cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE}
fi
rm -rf ${CACHE_TMP}/cache
fi
fi
fi
if [ ${POPULATE_CACHE} -eq 1 ]; then
if [ ${DEBUG} -eq 0 ];