Commit 6b2f531e authored by ossy@webkit.org's avatar ossy@webkit.org
Browse files

Add SVN mirror handling feature to build.webkit.org

https://bugs.webkit.org/show_bug.cgi?id=85887

Reviewed by Dirk Pranke.

* BuildSlaveSupport/build.webkit.org-config/config.json: Add SVN mirrors for Qt buildslaves hosted in Szeged.
* BuildSlaveSupport/build.webkit.org-config/master.cfg:
(ConfigureBuild.__init__): Add SVNMirror property to be able to watch which slaves use which mirror on build.webkit.org.
(ConfigureBuild.start):
(CheckOutSource.__init__): Set baseURL to the SVNMirror or the default http://svn.webkit.org/repository/webkit/
(WaitForSVNServer): Add new buildstep for waiting the SVNMirror to be in sync with http://svn.webkit.org/repository/webkit/
(Factory.__init__):
(BuildFactory.__init__):
(TestFactory.__init__):
(BuildAndTestFactory.__init__):
(BuildAndPerfTestFactory.__init__):
(BuildAndPerfTestWebKit2Factory.__init__):
(DownloadAndPerfTestFactory.__init__):
(DownloadAndPerfTestWebKit2Factory.__init__):
(loadBuilderConfig): Use kwargs instead of args to be able to add the optional SVNMirror factory argument.
* BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py:
(SVNMirrorTest):
(SVNMirrorTest.setUp): Load config.json once at startup.
(SVNMirrorTest.get_SVNMirrorFromConfig): Get the SVNMirror from config.json for a given buildslave.
(SVNMirrorTest.test_CheckOutSource): Compare CheckOutSource.baseURL with SVNMirror in config.json for all builders
* BuildSlaveSupport/build.webkit.org-config/wait-for-SVN-server.py: Added.
(getLatestSVNRevision): Get the latest SVN revison from the given server.
(waitForSVNRevision): Wait until the given SVN revision is committed to the given server. It doesn't wait if the
                      SVN revision is empty (force build triggered without revision) or the server is unavailable.

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@128399 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent 27e44406
......@@ -200,35 +200,42 @@
{
"name": "Qt Linux Release", "type": "BuildAndTest", "builddir": "qt-linux-release",
"platform": "qt-4.8", "configuration": "release", "architectures": ["i386"],
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"slavenames": ["szeged-linux-1"]
},
{
"name": "Qt Linux Release minimal", "type": "Build", "builddir": "qt-linux-release-minimal",
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"platform": "qt-4.8", "configuration": "release", "architectures": ["i386"],
"slavenames": ["szeged-linux-2"]
},
{
"name": "Qt Linux 64-bit Release (Perf)", "type": "BuildAndPerfTest", "builddir": "qt-linux-64-release-perf-tests",
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"platform": "qt-5.0-wk1", "configuration": "release", "architectures": ["x86_64"],
"slavenames": ["szeged-linux-3"]
},
{
"name": "Qt Linux 64-bit Release (WebKit2 Perf)", "type": "BuildAndPerfTestWebKit2", "builddir": "qt-linux-64-release-wk2-perf-tests",
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"platform": "qt-5.0-wk2", "configuration": "release", "architectures": ["x86_64"],
"slavenames": ["szeged-linux-5"]
},
{
"name": "Qt Linux ARMv7 Release", "type": "Build", "builddir": "qt-linux-armv7-release",
"SVNMirror": "svn://rain.inf.u-szeged.hu:3389/",
"platform": "qt-arm", "configuration": "release", "architectures": ["armv7"],
"slavenames": ["szeged-linux-4"]
},
{
"name": "Qt Windows 32-bit Release", "type": "Build", "builddir": "qt-windows-32bit-release",
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"platform": "qt-win", "configuration": "release", "architectures": ["i386"],
"slavenames": ["szeged-windows-1"]
},
{
"name": "Qt Windows 32-bit Debug", "type": "Build", "builddir": "qt-windows-32bit-debug",
"SVNMirror": "svn://rain.inf.u-szeged.hu/",
"platform": "qt-win", "configuration": "debug", "architectures": ["i386"],
"slavenames": ["szeged-windows-2"]
},
......
......@@ -92,14 +92,15 @@ class ConfigureBuild(buildstep.BuildStep):
name = "configure build"
description = ["configuring build"]
descriptionDone = ["configured build"]
def __init__(self, platform, configuration, architecture, buildOnly, *args, **kwargs):
def __init__(self, platform, configuration, architecture, buildOnly, SVNMirror, *args, **kwargs):
buildstep.BuildStep.__init__(self, *args, **kwargs)
self.platform = platform.split('-', 1)[0]
self.fullPlatform = platform
self.configuration = configuration
self.architecture = architecture
self.buildOnly = buildOnly
self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly)
self.SVNMirror = SVNMirror
self.addFactoryArguments(platform=platform, configuration=configuration, architecture=architecture, buildOnly=buildOnly, SVNMirror=SVNMirror)
def start(self):
self.setProperty("platform", self.platform)
......@@ -108,20 +109,26 @@ class ConfigureBuild(buildstep.BuildStep):
self.setProperty("architecture", self.architecture)
self.setProperty("buildOnly", self.buildOnly)
self.setProperty("shouldAbortEarly", True)
self.setProperty("SVNMirror", self.SVNMirror)
self.finished(SUCCESS)
return defer.succeed(None)
class CheckOutSource(source.SVN):
baseURL = "http://svn.webkit.org/repository/webkit/"
mode = "update"
def __init__(self, **kwargs):
kwargs['baseURL'] = self.baseURL
def __init__(self, SVNMirror, **kwargs):
kwargs['baseURL'] = SVNMirror or "http://svn.webkit.org/repository/webkit/"
kwargs['defaultBranch'] = "trunk"
kwargs['mode'] = self.mode
source.SVN.__init__(self, **kwargs)
self.addFactoryArguments(SVNMirror=SVNMirror)
class WaitForSVNServer(shell.ShellCommand):
name = "wait-for-svn-server"
command = ["python", "./Tools/BuildSlaveSupport/wait-for-SVN-server.py", "-r", WithProperties("%(revision)s"), "-s", WithProperties("%(SVNMirror)s")]
description = ["waiting for SVN server"]
descriptionDone = ["SVN server is ready"]
haltOnFailure = True
class InstallWin32Dependencies(shell.Compile):
description = ["installing dependencies"]
......@@ -697,10 +704,12 @@ class ExtractTestResultsAndLeaks(ExtractTestResults):
class Factory(factory.BuildFactory):
def __init__(self, platform, configuration, architectures, buildOnly):
def __init__(self, platform, configuration, architectures, buildOnly, SVNMirror):
factory.BuildFactory.__init__(self)
self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly))
self.addStep(CheckOutSource())
self.addStep(ConfigureBuild(platform=platform, configuration=configuration, architecture=" ".join(architectures), buildOnly=buildOnly, SVNMirror=SVNMirror))
if SVNMirror:
self.addStep(WaitForSVNServer())
self.addStep(CheckOutSource(SVNMirror=SVNMirror))
# There are multiple Qt slaves running on same machines, so buildslaves shouldn't kill the processes of other slaves.
if not platform.startswith("qt"):
self.addStep(KillOldProcesses())
......@@ -715,8 +724,8 @@ class Factory(factory.BuildFactory):
class BuildFactory(Factory):
def __init__(self, platform, configuration, architectures, triggers=None):
Factory.__init__(self, platform, configuration, architectures, True)
def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None):
Factory.__init__(self, platform, configuration, architectures, True, SVNMirror)
self.addStep(CompileWebKit())
if triggers:
self.addStep(ArchiveBuiltProduct())
......@@ -734,8 +743,8 @@ def pickLatestBuild(builder, requests):
class TestFactory(Factory):
TestClass = RunWebKitTests
ExtractTestResultsClass = ExtractTestResults
def __init__(self, platform, configuration, architectures):
Factory.__init__(self, platform, configuration, architectures, False)
def __init__(self, platform, configuration, architectures, SVNMirror=None):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror)
if platform.startswith("chromium"):
self.addStep(CleanupChromiumCrashLogs())
self.addStep(DownloadBuiltProduct())
......@@ -767,8 +776,8 @@ class TestFactory(Factory):
class BuildAndTestFactory(Factory):
TestClass = RunWebKitTests
ExtractTestResultsClass = ExtractTestResults
def __init__(self, platform, configuration, architectures, triggers=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
def __init__(self, platform, configuration, architectures, triggers=None, SVNMirror=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
if platform.startswith("chromium"):
self.addStep(CleanupChromiumCrashLogs)
self.addStep(CompileWebKit())
......@@ -811,31 +820,31 @@ class TestWebKit2Factory(TestFactory):
TestClass = RunWebKit2Tests
class BuildAndPerfTestFactory(Factory):
def __init__(self, platform, configuration, architectures, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
if platform.startswith("chromium"):
self.addStep(CleanupChromiumCrashLogs)
self.addStep(CompileWebKit())
self.addStep(RunAndUploadPerfTests())
class BuildAndPerfTestWebKit2Factory(Factory):
def __init__(self, platform, configuration, architectures, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
if platform.startswith("chromium"):
self.addStep(CleanupChromiumCrashLogs)
self.addStep(CompileWebKit())
self.addStep(RunAndUploadPerfTestsWebKit2())
class DownloadAndPerfTestFactory(Factory):
def __init__(self, platform, configuration, architectures, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
self.addStep(DownloadBuiltProduct())
self.addStep(ExtractBuiltProduct())
self.addStep(RunAndUploadPerfTests())
class DownloadAndPerfTestWebKit2Factory(Factory):
def __init__(self, platform, configuration, architectures, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, **kwargs)
def __init__(self, platform, configuration, architectures, SVNMirror=None, **kwargs):
Factory.__init__(self, platform, configuration, architectures, False, SVNMirror, **kwargs)
self.addStep(DownloadBuiltProduct)
self.addStep(ExtractBuiltProduct)
self.addStep(RunAndUploadPerfTestsWebKit2)
......@@ -902,13 +911,13 @@ def loadBuilderConfig(c):
builderType = builder.pop('type')
factory = globals()["%sFactory" % builderType]
factoryArgs = []
for key in "platform", "configuration", "architectures", "triggers":
factorykwargs = {}
for key in "platform", "configuration", "architectures", "triggers", "SVNMirror":
value = builder.pop(key, None)
if value:
factoryArgs.append(value)
factorykwargs[key] = value
builder["factory"] = factory(*factoryArgs)
builder["factory"] = factory(**factorykwargs)
if platform.startswith('chromium'):
builder["category"] = 'Chromium'
......
......@@ -332,6 +332,31 @@ Tests that timed out:
""")
class SVNMirrorTest(unittest.TestCase):
def setUp(self):
self.config = json.load(open('config.json'))
def get_SVNMirrorFromConfig(self, builderName):
SVNMirror = None
for builder in self.config['builders']:
if builder['name'] == builderName:
SVNMirror = builder.pop('SVNMirror', 'http://svn.webkit.org/repository/webkit/')
return SVNMirror
def test_CheckOutSource(self):
# SVN mirror feature isn't unittestable now with source.oldsource.SVN(==source.SVN) , only with source.svn.SVN(==SVN)
# https://bugs.webkit.org/show_bug.cgi?id=85887
if issubclass(CheckOutSource, source.SVN):
return
# Compare CheckOutSource.baseURL with SVNMirror (or with the default URL) in config.json for all builders
for builder in c['builders']:
for buildStepFactory, kwargs in builder['factory'].steps:
if str(buildStepFactory).split('.')[-1] == 'CheckOutSource':
CheckOutSourceInstance = buildStepFactory(**kwargs)
self.assertEquals(CheckOutSourceInstance.baseURL, self.get_SVNMirrorFromConfig(builder['name']))
class BuildStepsConstructorTest(unittest.TestCase):
# "Passing a BuildStep subclass to factory.addStep is deprecated. Please pass a BuildStep instance instead. Support will be dropped in v0.8.7."
# It checks if all builder's all buildsteps can be insantiated after migration.
......
#!/usr/bin/env python
#
# Copyright (C) 2006 John Pye
# Copyright (C) 2012 University of Szeged
#
# This script is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA
from optparse import OptionParser
import exceptions
import sys
import time
import xml.dom.minidom
import os
import subprocess
def getLatestSVNRevision(SVNServer):
try:
p = subprocess.Popen(["svn", "log", "--non-interactive", "--verbose", "--xml", "--limit=1", SVNServer], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
response = p.communicate()[0]
doc = xml.dom.minidom.parseString(response)
el = doc.getElementsByTagName("logentry")[0]
return el.getAttribute("revision")
except xml.parsers.expat.ExpatError, e:
print "FAILED TO PARSE 'svn log' XML:"
print str(e)
print "----"
print "RECEIVED TEXT:"
print response
sys.exit(1)
def waitForSVNRevision(SVNServer, revision):
if not revision:
latestRevision = int(getLatestSVNRevision(SVNServer))
print "Latest SVN revision on %s is r%d. Don't wait, because revision argument is empty." % (SVNServer, latestRevision)
return
while True:
latestRevision = int(getLatestSVNRevision(SVNServer))
if latestRevision < revision:
print "Latest SVN revision on %s is r%d, but we are waiting for r%d. Sleeping for 5 seconds." % (SVNServer, latestRevision, revision)
time.sleep(5)
else:
print "Latest SVN revision on %s is r%d, which is newer or equal than r%d." % (SVNServer, latestRevision, revision)
break
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-r", "--revision", dest="revision", type="int", help="SVN revision number")
parser.add_option("-s", "--svn-server", dest="SVNServer", help="SVN server")
options, args = parser.parse_args()
waitForSVNRevision(options.SVNServer, options.revision)
2012-09-12 Csaba Osztrogonác <ossy@webkit.org>
Add SVN mirror handling feature to build.webkit.org
https://bugs.webkit.org/show_bug.cgi?id=85887
Reviewed by Dirk Pranke.
* BuildSlaveSupport/build.webkit.org-config/config.json: Add SVN mirrors for Qt buildslaves hosted in Szeged.
* BuildSlaveSupport/build.webkit.org-config/master.cfg:
(ConfigureBuild.__init__): Add SVNMirror property to be able to watch which slaves use which mirror on build.webkit.org.
(ConfigureBuild.start):
(CheckOutSource.__init__): Set baseURL to the SVNMirror or the default http://svn.webkit.org/repository/webkit/
(WaitForSVNServer): Add new buildstep for waiting the SVNMirror to be in sync with http://svn.webkit.org/repository/webkit/
(Factory.__init__):
(BuildFactory.__init__):
(TestFactory.__init__):
(BuildAndTestFactory.__init__):
(BuildAndPerfTestFactory.__init__):
(BuildAndPerfTestWebKit2Factory.__init__):
(DownloadAndPerfTestFactory.__init__):
(DownloadAndPerfTestWebKit2Factory.__init__):
(loadBuilderConfig): Use kwargs instead of args to be able to add the optional SVNMirror factory argument.
* BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py:
(SVNMirrorTest):
(SVNMirrorTest.setUp): Load config.json once at startup.
(SVNMirrorTest.get_SVNMirrorFromConfig): Get the SVNMirror from config.json for a given buildslave.
(SVNMirrorTest.test_CheckOutSource): Compare CheckOutSource.baseURL with SVNMirror in config.json for all builders
* BuildSlaveSupport/build.webkit.org-config/wait-for-SVN-server.py: Added.
(getLatestSVNRevision): Get the latest SVN revison from the given server.
(waitForSVNRevision): Wait until the given SVN revision is committed to the given server. It doesn't wait if the
SVN revision is empty (force build triggered without revision) or the server is unavailable.
2012-09-12 Kangil Han <kangil.han@samsung.com>
 
[WK2][WTR] Set waitUntilDone watchdog timer value equal to WK1.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment