Adding initial build scripts based on RDO manager

Change-Id: Ib5f8ed4dfcaa42bcefcf2a7914c201a887beeacd
Signed-off-by: Dan Radez <dradez@redhat.com>

build/Makefile

+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# dradez@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+SHELL = /bin/bash
+############################################################################
+# BEGIN of variables to customize
+#
+#Input args
+export UNIT_TEST = FALSE
+export INTERACTIVE = TRUE
+export CENTDNLD = http://mirrors.cat.pdx.edu/centos/7.1.1503/isos/x86_64/CentOS-7-x86_64-DVD-1503-01.iso
+export ISOSRC = file:$(shell pwd)/CentOS-7-x86_64-DVD-1503-01.iso
+export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
+export PRODNO = "OPNFV_BGS"
+export REVSTATE = "P0000"
+export NEWISO = $(shell pwd)/release/OPNFV-CentOS-7-x86_64-${REVSTATE}.iso
+export GENESISRPM = $(shell pwd)/x86_64/opnfv-genesis-0.1-1.x86_64.rpm
+
+# Note! Invoke with "make REVSTATE=RXXXX all" to make release build!
+# Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location.
+
+#Build variables
+export BUILD_BASE := $(shell pwd)
+export CACHE_DIR := $(BUILD_BASE)/cache
+export VERSION_FILE := $(BUILD_BASE)/.versions
+export TOPDIR := $(shell pwd)
+
+CENTDIR := $(TOPDIR)/centiso
+#
+# END of variables to customize
+#############################################################################
+
+SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
+
+
+.PHONY: all
+all: iso
+	@echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
+	@echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+	@echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+
+############################################################################
+# BEGIN of Include definitions
+#
+include config.mk
+include cache.mk
+#
+# END Include definitions
+#############################################################################
+
+$(ISOCACHE):
+	test -s $(ISOCACHE) || { wget -nv $(CENTDNLD) ; }
+
+.PHONY: mount-centiso umount-centiso
+mount-centiso: $(ISOCACHE)
+	@echo "Mounting CentOS ISO in $(CENTDIR)"
+	@mkdir -p $(CENTDIR)
+	@fuseiso $(ISOCACHE) $(CENTDIR)
+
+umount-centiso:
+	@set +e
+	@echo "Unmounting CentOS ISO from $(CENTDIR)"
+	@fusermount -u $(CENTDIR)
+	@rmdir $(CENTDIR)
+	@set -e
+
+.PHONY: build-clean $(SUBCLEAN)
+build-clean: $(SUBCLEAN)
+	@rm -Rf centos
+	@rm -Rf release
+	@rm -Rf newiso
+	@rm -f $(NEWISO)
+
+.PHONY: clean $(SUBCLEAN)
+clean:  clean-cache $(SUBCLEAN)
+	@rm -f *.iso
+	@rm -Rf release
+	@rm -Rf newiso
+	@rm -f $(NEWISO)
+	@rm -f $(BUILD_BASE)/.versions
+
+$(SUBCLEAN): %.clean:
+	$(MAKE) -C $* -f Makefile clean
+
+.PHONY: rpm-clean
+rpm-clean:
+	rpmbuild --clean opnfv-genesis.spec -D '_topdir %(echo `pwd`)' -D '_builddir %(echo `pwd`)' -D '_sourcedir %(echo `pwd`)' -D '_rpmdir %(echo `pwd`)' -D '_specdir %(echo `pwd`)' -D '_srcrpmdir %(echo `pwd`)'
+
+.PHONY: rpm
+rpm:
+	pushd ../../ && git archive --format=tar --prefix=opnfv-genesis-0.1/ HEAD | gzip > foreman/build/opnfv-genesis.tar.gz
+	rpmbuild -ba opnfv-genesis.spec -D '_topdir %(echo `pwd`)' -D '_builddir %(echo `pwd`)' -D '_sourcedir %(echo `pwd`)' -D '_rpmdir %(echo `pwd`)' -D '_specdir %(echo `pwd`)' -D '_srcrpmdir %(echo `pwd`)' 
+	@make rpm-clean
+
+.PHONY: instack
+instack: instack.qcow2
+
+instack.qcow2:
+	@./instack.sh
+
+.PHONY: instack-clean
+instack-clean:
+	@virsh destroy instack 2> /dev/null || echo -n ''
+	@virsh undefine instack 2> /dev/null || echo -n ''
+	@virsh destroy baremetal_0 2> /dev/null || echo -n ''
+	@virsh undefine baremetal_0 2> /dev/null || echo -n ''
+	@virsh destroy baremetal_1 2> /dev/null || echo -n ''
+	@virsh undefine baremetal_1 2> /dev/null || echo -n ''
+	rm -f instack.xml
+	rm -f instack.qcow2
+	rm -rf stack
+
+
+# Todo: Make things smarter - we shouldn't need to clean everything
+# betwen make invocations.
+.PHONY: iso
+iso:	build-clean $(ISOCACHE) instack.qcow2 rpm
+	@make mount-centiso
+	@mkdir centos release
+	cp -r $(CENTDIR)/* centos
+	@make umount-centiso
+	# modify the installer iso's contents
+	@cp -f isolinux.cfg centos/isolinux/isolinux.cfg
+	@cp $(GENESISRPM) centos/Packages
+	# regenerate yum repo data
+	@echo "Generating new yum metadata"
+	createrepo --update -g ../c7-opnfv-x86_64-comps.xml centos
+	# build the iso
+	@echo "Building OPNFV iso"
+	mkisofs -b isolinux/isolinux.bin -no-emul-boot -boot-load-size 4 -boot-info-table -V "OPNFV CentOS 7 x86_64" -R -J -v -T -o $(NEWISO) centos
+	@printf "\n\nISO is built at $(NEWISO)\n\n"

build/cache.mk

+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+SHELL = /bin/bash
+CACHEVALIDATE := $(addsuffix .validate,$(SUBDIRS))
+CACHECLEAN := $(addsuffix .clean,$(CACHEFILES) $(CACHEDIRS))
+
+############################################################################
+# BEGIN of variables to customize
+#
+CACHEFILES += .versions
+CACHEFILES += $(shell basename $(ISOSRC))
+#
+# END of variables to customize
+############################################################################
+
+.PHONY: prepare-cache
+prepare-cache: make-cache-dir $(CACHEDIRS) $(CACHEFILES)
+
+.PHONY: make-cache-dir
+make-cache-dir:
+	@rm -rf ${CACHE_DIR}
+	@mkdir ${CACHE_DIR}
+
+.PHONY: clean-cache
+clean-cache: $(CACHECLEAN)
+	@rm -rf ${CACHE_DIR}
+
+.PHONY: $(CACHEDIRS)
+$(CACHEDIRS):
+	@mkdir -p $(dir $(CACHE_DIR)/$@)
+	@if [ ! -d $(BUILD_BASE)/$@ ]; then\
+	   mkdir -p $(BUILD_BASE)/$@;\
+	fi
+	@ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@
+
+.PHONY: $(CACHEFILES)
+$(CACHEFILES):
+	@mkdir -p $(dir $(CACHE_DIR)/$@)
+	@if [ ! -d $(dir $(BUILD_BASE)/$@) ]; then\
+	   mkdir -p $(dir $(BUILD_BASE)/$@);\
+	fi
+
+	@if [ ! -f $(BUILD_BASE)/$@ ]; then\
+	   echo " " > $(BUILD_BASE)/$@;\
+ 	   ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
+	   rm -f $(BUILD_BASE)/$@;\
+	else\
+	   ln -s $(BUILD_BASE)/$@ $(CACHE_DIR)/$@;\
+	fi
+
+.PHONY: validate-cache
+validate-cache: $(CACHEVALIDATE)
+	@if [[ $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep config.mk | awk '{print $$NF}') ]]; then\
+	   echo "Cache does not match current config.mk definition, cache must be rebuilt";\
+	   exit 1;\
+	fi;
+
+	@if [[ $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") != $(shell cat $(VERSION_FILE) | grep cache.mk | awk '{print $$NF}') ]]; then\
+	   echo "Cache does not match current cache.mk definition, cache must be rebuilt";\
+	   exit 1;\
+	fi;
+
+.PHONY: $(CACHEVALIDATE)
+$(CACHEVALIDATE): %.validate:
+	@echo VALIDATE $(CACHEVALIDATE)
+	$(MAKE) -C $* -f Makefile validate-cache
+
+.PHONY: $(CACHECLEAN)
+$(CACHECLEAN): %.clean:
+	rm -rf ${CACHE_DIR}/$*

build/instack.sh

+#!/bin/sh 
+set -e
+if ! id stack > /dev/null; then
+    useradd stack;
+    echo 'stack ALL=(root) NOPASSWD:ALL' | sudo tee -a /etc/sudoers.d/stack
+    echo 'Defaults:stack !requiretty' | sudo tee -a /etc/sudoers.d/stack
+    chmod 0440 /etc/sudoers.d/stack
+    echo 'Added user stack'
+fi
+
+if ! rpm -q epel-release > /dev/null; then
+    yum install http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+fi
+
+if ! rpm -q rdo-release > /dev/null; then
+    yum install -y https://rdoproject.org/repos/openstack-kilo/rdo-release-kilo.rpm
+fi
+
+if ! [ -a /etc/yum.repos.d/rdo-management-trunk.repo ]; then
+    curl -o /etc/yum.repos.d/rdo-management-trunk.repo http://trunk-mgt.rdoproject.org/centos-kilo/current-passed-ci/delorean-rdo-management.repo
+fi
+
+if ! rpm -q instack-undercloud > /dev/null; then
+    yum install -y instack-undercloud
+fi
+
+sudo -u stack -- sh -c 'cd; instack-virt-setup'
+#if ! ovs-vsctl show | grep brbm; then
+#    ovs-vsctl add-port brbm em2
+#fi
+cp /home/stack/.ssh/id_rsa* /root/.ssh/
+UNDERCLOUD=$(virsh net-dhcp-leases default | grep instack | awk '{print $5}' | awk -F '/' '{print $1}')
+
+ssh -T -o "StrictHostKeyChecking no" root@$UNDERCLOUD <<EOI
+if ! rpm -q rdo-release > /dev/null; then
+    yum install -y https://rdoproject.org/repos/openstack-kilo/rdo-release-kilo.rpm
+fi
+
+if ! [ -a /etc/yum.repos.d/rdo-management-trunk.repo ]; then
+    curl -o /etc/yum.repos.d/rdo-management-trunk.repo http://trunk-mgt.rdoproject.org/centos-kilo/current-passed-ci/delorean-rdo-management.repo
+fi
+
+yum install -y python-rdomanager-oscplugin
+cp /root/.ssh/authorized_keys /home/stack/.ssh/authorized_keys
+chown stack:stack /home/stack/.ssh/authorized_keys
+EOI
+
+ssh -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "openstack undercloud install"
+#ssh -T -o "StrictHostKeyChecking no" stack@$UNDERCLOUD <<EOI
+#echo "Running undercloud install"
+#openstack undercloud install
+#dontexit=\$(openstack undercloud install)
+#openstack undercloud install && sudo halt -p
+#EOI
+echo "Shuttind down instack to take snapshop"
+virsh shutdown instack
+
+echo "Waiting for instack VM to shutdown"
+while virsh list | grep instack; do
+    echo -n "."
+    sleep 5
+done
+
+echo "Copying instack disk image and starting instack VM."
+cp -f /var/lib/libvirt/images/instack.qcow2 .
+virsh dumpxml instack > instack.xml
+virsh vol-dumpxml instack.qcow2 --pool default > instack.qcow2.xml
+virsh start instack
+
+echo "Waiting for instack VM to start"
+while ! ping -c 1 $UNDERCLOUD > /dev/null; do
+    echo -n "."
+    sleep 5
+done
+while ! ssh -T -o "StrictHostKeyChecking no" root@$UNDERCLOUD "echo ''" > /dev/null; do
+    echo -n "."
+    sleep 3
+done
+
+echo "Copying CentOS Cache to instack VM"
+ssh -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "mkdir .cache"
+scp -r /home/stack/.cache/image-create/CentOS-7-x86_64-GenericCloud* stack@$UNDERCLOUD:.cache/
+
+echo "Building overcloud images"
+ssh -tt -o "StrictHostKeyChecking no" stack@$UNDERCLOUD "openstack overcloud image build --all"
+
+echo "Copying overcloud images"
+mkdir stack
+scp stack@$UNDERCLOUD:deploy-ramdisk-ironic.initramfs stack
+scp stack@$UNDERCLOUD:deploy-ramdisk-ironic.kernel stack
+scp stack@$UNDERCLOUD:discovery-ramdisk.initramfs stack
+scp stack@$UNDERCLOUD:discovery-ramdisk.kernel stack
+scp stack@$UNDERCLOUD:fedora-user.qcow2 stack
+scp stack@$UNDERCLOUD:overcloud-full.initrd stack
+scp stack@$UNDERCLOUD:overcloud-full.qcow2 stack
+scp stack@$UNDERCLOUD:overcloud-full.vmlinuz stack

ci/build.sh

+#!/bin/bash
+set -e
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# dradez@redhat.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+trap 'echo "Exiting ..."; \
+if [ -f ${LOCK_FILE} ]; then \
+   if [ $(cat ${LOCK_FILE}) -eq $$ ]; then \
+      rm -f ${LOCK_FILE}; \
+   fi; \
+fi;' EXIT
+
+############################################################################
+# BEGIN of usage description
+#
+usage ()
+{
+cat << EOF
+$0 Builds the Foreman OPNFV Deployment ISO
+
+usage: $0 [-s spec-file] [-c cache-URI] [-l log-file] [-f Flags] build-directory
+
+OPTIONS:
+  -s spec-file ($BUILD_SPEC), define the build-spec file, default ../build/config.mk
+  -c cache base URI ($BUILD_CACHE_URI), specifies the base URI to a build cache to be used/updated - the name is automatically generated from the md5sum of the spec-file, http://, ftp://, file://[absolute path] suported.
+
+  -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal
+  -v version tag to be applied to the build result
+  -r alternative remote access method script/program. curl is default.
+  -t run small build-script unit test.
+  -T run large build-script unit test.
+  -f build flags ($BUILD_FLAGS):
+     o s: Do nothing, succeed
+     o f: Do nothing, fail
+     o t: run build unit tests
+     o i: run interactive (-t flag to docker run)
+     o P: Populate a new local cache and push it to the (-c cache-URI) cache artifactory if -c option is present, currently file://, http:// and ftp:// are supported
+     o d: Detatch - NOT YET SUPPORTED
+
+  build-directory ($BUILD_DIR), specifies the directory for the output artifacts (.iso file).
+
+  -h help, prints this help text
+
+Description:
+build.sh builds opnfv .iso artifact.
+To reduce build time it uses build cache on a local or remote location. The cache is rebuilt and uploaded if either of the below conditions are met:
+1) The P(opulate) flag is set and the -c cache-base-URI is provided, if -c is not provided the cache will stay local.
+2) If the cache is invalidated by one of the following conditions:
+   - The config spec md5sum does not compare to the md5sum for the spec which the cache was built.
+   - The git Commit-Id on the remote repos/HEAD defined in the spec file does not correspont with the Commit-Id for what the cache was built with.
+3) A valid cache does not exist on the specified -c cache-base-URI.
+
+The cache URI object name is foreman_cache-"md5sum(spec file)"
+
+Logging by default to console, but can be directed elsewhere with the -l option in which case both stdout and stderr is redirected to that destination.
+
+Built in unit testing of components is enabled by adding the t(est) flag.
+
+Return codes:
+ - 0 Success!
+ - 1-99 Unspecified build error
+ - 100-199 Build system internal error (not build it self)
+   o 101 Build system instance busy
+ - 200 Build failure
+
+Examples:
+build -c http://opnfv.org/artifactory/foreman/cache -d ~/jenkins/genesis/foreman/ci/output -f ti
+NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work
+EOF
+}
+#
+# END of usage description
+############################################################################
+
+############################################################################
+# BEGIN of variables to customize
+#
+BUILD_BASE=$(readlink -e ../build/)
+RESULT_DIR="${BUILD_BASE}/release"
+BUILD_SPEC="${BUILD_BASE}/config.mk"
+CACHE_DIR="cache"
+LOCAL_CACHE_ARCH_NAME="foreman-cache"
+REMOTE_CACHE_ARCH_NAME="foreman_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")"
+REMOTE_ACCESS_METHD=curl
+INCLUDE_DIR=../include
+#
+# END of variables to customize
+############################################################################
+
+############################################################################
+# BEGIN of script assigned variables
+#
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+LOCK_FILE="${SCRIPT_DIR}/.build.lck"
+CACHE_TMP="${SCRIPT_DIR}/tmp"
+TEST_SUCCEED=0
+TEST_FAIL=0
+UNIT_TEST=0
+UPDATE_CACHE=0
+POPULATE_CACHE=0
+RECURSIV=0
+DETACH=0
+DEBUG=0
+INTEGRATION_TEST=0
+FULL_INTEGRATION_TEST=0
+INTERACTIVE=0
+BUILD_CACHE_URI=
+BUILD_SPEC=
+BUILD_DIR=
+BUILD_LOG=
+BUILD_VERSION=
+MAKE_ARGS=
+#
+# END of script assigned variables
+############################################################################
+
+############################################################################
+# BEGIN of include pragmas
+#
+source ${INCLUDE_DIR}/build.sh.debug
+#
+# END of include
+############################################################################
+
+############################################################################
+# BEGIN of main
+#
+while getopts "s:c:d:v:f:l:r:RtTh" OPTION
+do
+    case $OPTION in
+	h)
+	    usage
+	    rc=0
+	    exit $rc
+	    ;;
+
+	s)
+	    BUILD_SPEC=${OPTARG}
+	    ;;
+
+	c)
+	    BUILD_CACHE_URI=${OPTARG}
+	    ;;
+
+	d)
+	    BUILD_DIR=${OPTARG}
+	    ;;
+
+	l)
+	    BUILD_LOG=${OPTARG}
+	    ;;
+
+	v)
+	    BUILD_VERSION=${OPTARG}
+	    ;;
+
+	f)
+	    BUILD_FLAGS=${OPTARG}
+	    ;;
+
+	r)  REMOTE_ACCESS_METHD=${OPTARG}
+	    ;;
+
+	R)
+	    RECURSIVE=1
+	    ;;
+
+	t)
+	    INTEGRATION_TEST=1
+	    ;;
+
+	T)
+	    INTEGRATION_TEST=1
+	    FULL_INTEGRATION_TEST=1
+	    ;;
+
+	*)
+	    echo "${OPTION} is not a valid argument"
+	    rc=100
+	    exit $rc
+	    ;;
+    esac
+done
+
+if [ -z $BUILD_DIR ]; then
+    BUILD_DIR=$(echo $@ | cut -d ' ' -f ${OPTIND})
+fi
+
+for ((i=0; i<${#BUILD_FLAGS};i++)); do
+    case ${BUILD_FLAGS:$i:1} in
+	s)
+	    rc=0
+	    exit $rc
+	    ;;
+
+	f)
+	    rc=1
+	    exit $rc
+	    ;;
+
+	t)
+	    UNIT_TEST=1
+	    ;;
+
+	i)
+	    INTERACTIVE=1
+	    ;;
+
+	P)
+	    POPULATE_CACHE=1
+	    ;;
+
+	d)
+	    DETACH=1
+	    echo "Detach is not yet supported - exiting ...."
+	    rc=100
+	    exit $rc
+	    ;;
+
+	D)
+ 	    DEBUG=1
+	    ;;
+
+	*)
+	    echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...."
+	    rc=100
+	    exit $rc
+	    ;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+if [ ${INTEGRATION_TEST} -eq 1 ]; then
+    integration-test
+    rc=0
+    exit $rc
+fi
+
+if [ ! -f ${BUILD_SPEC} ]; then
+    echo "spec file does not exist: $BUILD_SPEC - exiting ...."
+    rc=100
+    exit $rc
+fi
+
+if [ -z ${BUILD_DIR} ]; then
+    echo "Missing build directory - exiting ...."
+    rc=100
+    exit $rc
+fi
+
+if [ ! -z ${BUILD_LOG} ]; then
+    if [[ ${RECURSIVE} -ne 1 ]]; then
+	set +e
+	eval $0 -R $@ > ${BUILD_LOG} 2>&1
+	rc=$?
+	set -e
+	if [ $rc -ne 0]; then
+	    exit $rc
+	fi
+    fi
+fi
+
+if [ ${TEST_SUCCEED} -eq 1 ]; then
+    sleep 1
+    rc=0
+    exit $rc
+fi
+
+if [ ${TEST_FAIL} -eq 1 ]; then
+    sleep 1
+    rc=1
+    exit $rc
+fi
+
+if [ -e ${LOCK_FILE} ]; then
+    echo "A build job is already running, exiting....."
+    rc=101
+    exit $rc
+fi
+
+echo $$ > ${LOCK_FILE}
+
+if [ ! -z ${BUILD_CACHE_URI} ]; then
+    if [ ${POPULATE_CACHE} -ne 1 ]; then
+	rm -rf ${CACHE_TMP}/cache
+	mkdir -p ${CACHE_TMP}/cache
+	echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..."
+	set +e
+	${REMOTE_ACCESS_METHD} -o ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
+	rc=$?
+	set -e
+	if [ $rc -ne 0 ]; then
+		echo "Remote cache does not exist, or is not accessible - a new cache will be built ..."
+		POPULATE_CACHE=1
+	else
+	    echo "Unpacking cache file ..."
+	    tar -C ${CACHE_TMP}/cache -xvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz
+	    cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/.
+	    set +e
+       	    make -C ${BUILD_BASE} validate-cache;
+	    rc=$?
+	    set -e
+
+	    if [ $rc -ne 0 ]; then
+		echo "Cache invalid - a new cache will be built "
+		POPULATE_CACHE=1
+	    else
+		cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE}
+	    fi
+	    rm -rf ${CACHE_TMP}/cache
+	fi
+    fi
+fi
+
+if [ ${POPULATE_CACHE} -eq 1 ]; then
+    if [ ${DEBUG} -eq 0 ];