Commit befdb0cd authored by dglazkov@chromium.org's avatar dglazkov@chromium.org
Browse files

garden-o-matic should extrapolate failures given a set of known results

https://bugs.webkit.org/show_bug.cgi?id=65882

Reviewed by Adam Barth.

* Scripts/webkitpy/layout_tests/models/test_configuration.py: Spuriously covered my old sins.
* Scripts/webkitpy/layout_tests/port/builders.py: Augmented builders dictionary to contain build coverage specifiers,
    adjusted all call sites.
* Scripts/webkitpy/tool/servers/gardeningserver.py: Added BuildCoverageExtrapolator, a thing that
    makes wider net for specific failures.
* Scripts/webkitpy/tool/servers/gardeningserver_unittest.py: Added unit tests.

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@92770 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent 969b4521
2011-08-09 Dimitri Glazkov <dglazkov@chromium.org>
garden-o-matic should extrapolate failures given a set of known results
https://bugs.webkit.org/show_bug.cgi?id=65882
Reviewed by Adam Barth.
* Scripts/webkitpy/layout_tests/models/test_configuration.py: Spuriously covered my old sins.
* Scripts/webkitpy/layout_tests/port/builders.py: Augmented builders dictionary to contain build coverage specifiers,
adjusted all call sites.
* Scripts/webkitpy/tool/servers/gardeningserver.py: Added BuildCoverageExtrapolator, a thing that
makes wider net for specific failures.
* Scripts/webkitpy/tool/servers/gardeningserver_unittest.py: Added unit tests.
2011-08-10 Adam Barth <abarth@webkit.org>
 
The URL we were computing for retrieving baselines was missing a slash
......@@ -65,7 +65,7 @@ class TestConfiguration(object):
return self.__dict__.values()
class SpecifierSorter:
class SpecifierSorter(object):
def __init__(self, all_test_configurations=None, macros=None):
self._specifier_to_category = {}
......@@ -109,7 +109,7 @@ class SpecifierSorter:
return reduce(sort_and_return, category_slots, [])
class TestConfigurationConverter:
class TestConfigurationConverter(object):
def __init__(self, all_test_configurations, configuration_macros=None):
self._all_test_configurations = all_test_configurations
self._configuration_macros = configuration_macros or {}
......
......@@ -32,39 +32,42 @@ import re
from webkitpy.common.memoized import memoized
# In this dictionary, each item stores:
# * port_name -- a fully qualified port name
# * specifiers -- a set of specifiers, representing configurations covered by this builder.
_exact_matches = {
# These builders are on build.chromium.org.
"Webkit Win": "chromium-win-xp",
"Webkit Vista": "chromium-win-vista",
"Webkit Win7": "chromium-win-win7",
"Webkit Win (dbg)(1)": "chromium-win-xp",
"Webkit Win (dbg)(2)": "chromium-win-xp",
"Webkit Linux": "chromium-linux-x86_64",
"Webkit Linux 32": "chromium-linux-x86",
"Webkit Linux (dbg)(1)": "chromium-linux-x86_64",
"Webkit Linux (dbg)(2)": "chromium-linux-x86_64",
"Webkit Mac10.5": "chromium-mac-leopard",
"Webkit Mac10.5 (dbg)(1)": "chromium-mac-leopard",
"Webkit Mac10.5 (dbg)(2)": "chromium-mac-leopard",
"Webkit Mac10.6": "chromium-mac-snowleopard",
"Webkit Mac10.6 (dbg)": "chromium-mac-snowleopard",
"Webkit Mac10.6 - GPU": "chromium-gpu-mac-snowleopard",
"Webkit Win - GPU": "chromium-gpu-win-xp",
"Webkit Win7 - GPU": "chromium-gpu-win-win7",
"Webkit Win": {"port_name": "chromium-win-xp", "specifiers": set(["xp", "release", "cpu"])},
"Webkit Vista": {"port_name": "chromium-win-vista", "specifiers": set(["vista"])},
"Webkit Win7": {"port_name": "chromium-win-win7", "specifiers": set(["win7", "cpu"])},
"Webkit Win (dbg)(1)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])},
"Webkit Win (dbg)(2)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])},
"Webkit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])},
"Webkit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])},
"Webkit Linux (dbg)(1)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])},
"Webkit Linux (dbg)(2)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])},
"Webkit Mac10.5": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard"])},
"Webkit Mac10.5 (dbg)(1)": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard", "debug"])},
"Webkit Mac10.5 (dbg)(2)": {"port_name": "chromium-mac-leopard", "specifiers": set(["leopard", "debug"])},
"Webkit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])},
"Webkit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])},
"Webkit Mac10.6 - GPU": {"port_name": "chromium-gpu-mac-snowleopard", "specifiers": set(["snowleopard", "gpu"])},
"Webkit Win - GPU": {"port_name": "chromium-gpu-win-xp", "specifiers": set(["xp", "release", "gpu"])},
"Webkit Win7 - GPU": {"port_name": "chromium-gpu-win-win7", "specifiers": set(["win7", "vista", "release", "gpu"])},
# FIXME: For some reason, these port names don't work correctly.
# "Webkit Linux - GPU": "chromium-gpu-linux-x86_64",
# "Webkit Linux 32 - GPU": "chromium-gpu-linux-x86",
"Webkit Mac10.5 - GPU": "chromium-gpu-mac-leopard",
"Webkit Mac10.6 - GPU": "chromium-gpu-mac-snowleopard",
# "Webkit Linux - GPU": {"port_name": "chromium-gpu-linux-x86_64", "specifiers": set(["linux", "gpu"])},
# "Webkit Linux 32 - GPU": {"port_name": "chromium-gpu-linux-x86", "specifiers": set(["linux", "x86", "gpu"])},
"Webkit Mac10.5 - GPU": {"port_name": "chromium-gpu-mac-leopard", "specifiers": set(["leopard", "gpu"])},
"Webkit Mac10.6 - GPU": {"port_name": "chromium-gpu-mac-snowleopard", "specifiers": set(["mac", "gpu"])},
# These builders are on build.webkit.org.
"GTK Linux 32-bit Debug": "gtk",
"Leopard Intel Debug (Tests)": "mac-leopard",
"SnowLeopard Intel Release (Tests)": "mac-snowleopard",
"SnowLeopard Intel Release (WebKit2 Tests)": "mac-wk2",
"Qt Linux Release": "qt-linux",
"Windows XP Debug (Tests)": "win-xp",
"Windows 7 Release (WebKit2 Tests)": "win-wk2",
"GTK Linux 32-bit Debug": {"port_name": "gtk", "specifiers": set(["gtk"])},
"Leopard Intel Debug (Tests)": {"port_name": "mac-leopard", "specifiers": set(["leopard"])},
"SnowLeopard Intel Release (Tests)": {"port_name": "mac-snowleopard", "specifiers": set(["snowleopard"])},
"SnowLeopard Intel Release (WebKit2 Tests)": {"port_name": "mac-wk2", "specifiers": set(["wk2"])},
"Qt Linux Release": {"port_name": "qt-linux", "specifiers": set(["win", "linux", "mac"])},
"Windows XP Debug (Tests)": {"port_name": "win-xp", "specifiers": set(["win"])},
"Windows 7 Release (WebKit2 Tests)": {"port_name": "win-wk2", "specifiers": set(["wk2"])},
}
......@@ -82,7 +85,7 @@ _fuzzy_matches = {
_ports_without_builders = [
# FIXME: Including chromium-gpu-linux below is a workaroudn for
# FIXME: Including chromium-gpu-linux below is a workaround for
# chromium-gpu-linux-x86_64 and chromium-gpu-linux-x86 not working properly.
"chromium-gpu-linux",
"google-chrome-linux32",
......@@ -97,14 +100,23 @@ def builder_path_from_name(builder_name):
return re.sub(r'[\s().]', '_', builder_name)
@memoized
def all_builder_names():
return sorted(set(_exact_matches.keys()))
@memoized
def all_port_names():
return sorted(set(_exact_matches.values() + _ports_without_builders))
return sorted(set(map(lambda x: x["port_name"], _exact_matches.values()) + _ports_without_builders))
def coverage_specifiers_for_builder_name(builder_name):
return _exact_matches[builder_name].get("specifiers", set())
def port_name_for_builder_name(builder_name):
if builder_name in _exact_matches:
return _exact_matches[builder_name]
return _exact_matches[builder_name]["port_name"]
for regexp, port_name in _fuzzy_matches.items():
if re.match(regexp, builder_name):
......@@ -112,8 +124,8 @@ def port_name_for_builder_name(builder_name):
def builder_name_for_port_name(target_port_name):
for builder_name, port_name in _exact_matches.items():
if port_name == target_port_name:
for builder_name, builder_info in _exact_matches.items():
if builder_info["port_name"] == target_port_name:
return builder_name
return None
......
......@@ -30,19 +30,32 @@ from webkitpy.tool.servers.reflectionhandler import ReflectionHandler
from webkitpy.layout_tests.models.test_expectations import BugManager, TestExpectationParser, TestExpectations, TestExpectationsEditor, TestExpectationSerializer
from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter
from webkitpy.layout_tests.port import factory
from webkitpy.layout_tests.port import builders
class BuildCoverageExtrapolator(object):
def __init__(self, test_configuration_converter):
self._test_configuration_converter = test_configuration_converter
@memoized
def _covered_test_configurations_for_builder_name(self):
coverage = {}
for builder_name in builders.all_builder_names():
coverage[builder_name] = self._test_configuration_converter.to_config_set(builders.coverage_specifiers_for_builder_name(builder_name))
return coverage
def extrapolate_test_configurations(self, builder_name):
return self._covered_test_configurations_for_builder_name()[builder_name]
class GardeningExpectationsUpdater(BugManager):
def __init__(self, tool, port):
self._converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
self._extrapolator = BuildCoverageExtrapolator(self._converter)
self._parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False)
self._path_to_test_expectations_file = port.path_to_test_expectations_file()
self._tool = tool
@memoized
def _builder_to_test_config(self, builder_name):
return factory.get_from_builder_name(builder_name).test_configuration()
def close_bug(self, bug_id, reference_bug_id=None):
# FIXME: Implement this properly.
pass
......@@ -56,12 +69,15 @@ class GardeningExpectationsUpdater(BugManager):
self._parser.parse(expectation_line)
editor = TestExpectationsEditor(expectation_lines, self)
updated_expectation_lines = []
# FIXME: Group failures by testName+failureTypeList.
for failure_info in failure_info_list:
expectation_set = set(filter(None, map(TestExpectations.expectation_from_string, failure_info['failureTypeList'])))
assert(expectation_set)
test_name = failure_info['testName']
assert(test_name)
updated_expectation_lines.extend(editor.update_expectation(test_name, set([self._builder_to_test_config(failure_info['builderName'])]), expectation_set))
builder_name = failure_info['builderName']
affected_test_configuration_set = self._extrapolator.extrapolate_test_configurations(builder_name)
updated_expectation_lines.extend(editor.update_expectation(test_name, affected_test_configuration_set, expectation_set))
self._tool.filesystem.write_text_file(self._path_to_test_expectations_file, TestExpectationSerializer.list_to_string(expectation_lines, self._converter, reconstitute_only_these=updated_expectation_lines))
......
......@@ -36,6 +36,7 @@ import unittest
from webkitpy.common.system.outputcapture import OutputCapture
from webkitpy.layout_tests.port import factory
from webkitpy.layout_tests.models.test_configuration import *
from webkitpy.thirdparty.mock import Mock
from webkitpy.tool.mocktool import MockTool, MockExecutive
from webkitpy.tool.servers.gardeningserver import *
......@@ -82,6 +83,21 @@ class TestGardeningHTTPRequestHandler(GardeningHTTPRequestHandler):
print "== End JSON Response =="
class BuildCoverageExtrapolatorTest(unittest.TestCase):
def test_extrapolate(self):
# FIXME: Make this test not rely on actual (not mock) port objects.
port = factory.get('chromium-win-win7', None)
converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
extrapolator = BuildCoverageExtrapolator(converter)
self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release', graphics_type='cpu')]))
self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Vista"), set([
TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='cpu'),
TestConfiguration(version='vista', architecture='x86', build_type='debug', graphics_type='gpu'),
TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='gpu'),
TestConfiguration(version='vista', architecture='x86', build_type='release', graphics_type='cpu')]))
self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato")
class GardeningExpectationsUpdaterTest(unittest.TestCase):
def __init__(self, testFunc):
self.tool = MockTool()
......@@ -107,7 +123,7 @@ class GardeningExpectationsUpdaterTest(unittest.TestCase):
def test_unknown_builder(self):
failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Bob", "failureTypeList": ["IMAGE"]}]
self.assert_update(failure_info_list, expected_exception=AssertionError)
self.assert_update(failure_info_list, expected_exception=KeyError)
def test_empty_failure_type_list(self):
failure_info_list = [{"testName": "failures/expected/image.html", "builderName": "Webkit Win", "failureTypeList": []}]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment