base.py 35.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the Google name nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Abstract base class of Port-specific entrypoints for the layout tests
test infrastructure (the Port and Driver classes)."""

import cgi
import difflib
import errno
import os
37
import shlex
38
import sys
39
import time
40
41

import apache_http_server
42
import config as port_config
43
import http_lock
44
import http_server
45
import test_files
46
47
import websocket_server

48
49
from webkitpy.common import system
from webkitpy.common.system import filesystem
50
from webkitpy.common.system import logutils
51
from webkitpy.common.system import path
52
from webkitpy.common.system.executive import Executive, ScriptError
53
from webkitpy.common.system.user import User
54
55
56
57


_log = logutils.get_logger(__file__)

58

59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
class DummyOptions(object):
    """Fake implementation of optparse.Values. Cloned from
    webkitpy.tool.mocktool.MockOptions.

    """

    def __init__(self, **kwargs):
        # The caller can set option values using keyword arguments. We don't
        # set any values by default because we don't know how this
        # object will be used. Generally speaking unit tests should
        # subclass this or provider wrapper functions that set a common
        # set of options.
        for key, value in kwargs.items():
            self.__dict__[key] = value


75
# FIXME: This class should merge with webkitpy.webkit_port at some point.
76
class Port(object):
77
78
79
80
81
82
83
84
85
86
    """Abstract class for Port-specific hooks for the layout_test package."""

    def __init__(self, port_name=None, options=None,
                 executive=None,
                 user=None,
                 filesystem=None,
                 config=None,
                 **kwargs):
        self._name = port_name
        self._options = options
87
88
89
90
91
        if self._options is None:
            # FIXME: Ideally we'd have a package-wide way to get a
            # well-formed options object that had all of the necessary
            # options defined on it.
            self._options = DummyOptions()
92
93
94
95
96
        self._executive = executive or Executive()
        self._user = user or User()
        self._filesystem = filesystem or system.filesystem.FileSystem()
        self._config = config or port_config.Config(self._executive,
                                                    self._filesystem)
97
98
99
100
        self._helper = None
        self._http_server = None
        self._webkit_base_dir = None
        self._websocket_server = None
101
        self._http_lock = None
102

103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        # Python's Popen has a bug that causes any pipes opened to a
        # process that can't be executed to be leaked.  Since this
        # code is specifically designed to tolerate exec failures
        # to gracefully handle cases where wdiff is not installed,
        # the bug results in a massive file descriptor leak. As a
        # workaround, if an exec failure is ever experienced for
        # wdiff, assume it's not available.  This will leak one
        # file descriptor but that's better than leaking each time
        # wdiff would be run.
        #
        # http://mail.python.org/pipermail/python-list/
        #    2008-August/505753.html
        # http://bugs.python.org/issue3210
        self._wdiff_available = True

        self._pretty_patch_path = self.path_from_webkit_base("BugsSite",
119
            "PrettyPatch", "prettify.rb")
120
        self._pretty_patch_available = True
121
122
123
        self.set_option_default('configuration', None)
        if self._options.configuration is None:
            self._options.configuration = self.default_configuration()
124

125
    def default_child_processes(self):
126
127
128
        """Return the number of DumpRenderTree instances to use for this
        port."""
        return self._executive.cpu_count()
129
130
131
132
133
134
135
136
137
138
139

    def baseline_path(self):
        """Return the absolute path to the directory to store new baselines
        in for this port."""
        raise NotImplementedError('Port.baseline_path')

    def baseline_search_path(self):
        """Return a list of absolute paths to directories to search under for
        baselines. The directories are searched in order."""
        raise NotImplementedError('Port.baseline_search_path')

140
141
142
143
144
    def check_build(self, needs_http):
        """This routine is used to ensure that the build is up to date
        and all the needed binaries are present."""
        raise NotImplementedError('Port.check_build')

145
    def check_sys_deps(self, needs_http):
146
        """If the port needs to do some runtime checks to ensure that the
147
148
        tests can be run successfully, it should override this routine.
        This step can be skipped with --nocheck-sys-deps.
149
150

        Returns whether the system is properly configured."""
151
        return True
152

153
154
    def check_image_diff(self, override_step=None, logging=True):
        """This routine is used to check whether image_diff binary exists."""
155
        raise NotImplementedError('Port.check_image_diff')
156

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
    def check_pretty_patch(self):
        """Checks whether we can use the PrettyPatch ruby script."""

        # check if Ruby is installed
        try:
            result = self._executive.run_command(['ruby', '--version'])
        except OSError, e:
            if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
                _log.error("Ruby is not installed; "
                           "can't generate pretty patches.")
                _log.error('')
                return False

        if not self.path_exists(self._pretty_patch_path):
            _log.error('Unable to find %s .' % self._pretty_patch_path)
            _log.error("Can't generate pretty patches.")
            _log.error('')
            return False

        return True

178
    def compare_text(self, expected_text, actual_text):
179
180
181
182
183
        """Return whether or not the two strings are *not* equal. This
        routine is used to diff text output.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
184
        return expected_text != actual_text
185

186
    def diff_image(self, expected_contents, actual_contents,
187
                   diff_filename=None, tolerance=0):
188
        """Compare two images and produce a delta image file.
189

190
        Return True if the two images are different, False if they are the same.
191
        Also produce a delta image of the two images and write that into
192
        |diff_filename| if it is not None.
193

194
195
196
        |tolerance| should be a percentage value (0.0 - 100.0).
        If it is omitted, the port default tolerance value is used.

197
198
        """
        raise NotImplementedError('Port.diff_image')
199

200

201
202
    def diff_text(self, expected_text, actual_text,
                  expected_filename, actual_filename):
203
204
205
206
207
        """Returns a string containing the diff of the two text strings
        in 'unified diff' format.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
208
209
210
211
212
213
214
215
216
217

        # The filenames show up in the diff output, make sure they're
        # raw bytes and not unicode, so that they don't trigger join()
        # trying to decode the input.
        def to_raw_bytes(str):
            if isinstance(str, unicode):
                return str.encode('utf-8')
            return str
        expected_filename = to_raw_bytes(expected_filename)
        actual_filename = to_raw_bytes(actual_filename)
218
219
220
221
222
223
        diff = difflib.unified_diff(expected_text.splitlines(True),
                                    actual_text.splitlines(True),
                                    expected_filename,
                                    actual_filename)
        return ''.join(diff)

224
225
226
227
228
229
230
    def driver_name(self):
        """Returns the name of the actual binary that is performing the test,
        so that it can be referred to in log messages. In most cases this
        will be DumpRenderTree, but if a port uses a binary with a different
        name, it can be overridden here."""
        return "DumpRenderTree"

231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
    def expected_baselines(self, filename, suffix, all_baselines=False):
        """Given a test name, finds where the baseline results are located.

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g.
            '.txt' or '.png'.  This should not be None, but may be an empty
            string.
        all_baselines: If True, return an ordered list of all baseline paths
            for the given platform. If False, return only the first one.
        Returns
        a list of ( platform_dir, results_filename ), where
            platform_dir - abs path to the top of the results tree (or test
                tree)
            results_filename - relative path from top of tree to the results
                file
            (os.path.join of the two gives you the full path to the file,
                unless None was returned.)
        Return values will be in the format appropriate for the current
        platform (e.g., "\\" for path separators on Windows). If the results
        file is not found, then None will be returned for the directory,
        but the expected relative pathname will still be returned.

        This routine is generic but lives here since it is used in
        conjunction with the other baseline and filename routines that are
        platform specific.
        """
        testname = os.path.splitext(self.relative_test_filename(filename))[0]

        baseline_filename = testname + '-expected' + suffix

        baseline_search_path = self.baseline_search_path()

        baselines = []
        for platform_dir in baseline_search_path:
266
267
            if self.path_exists(self._filesystem.join(platform_dir,
                                                      baseline_filename)):
268
269
270
271
272
273
274
275
                baselines.append((platform_dir, baseline_filename))

            if not all_baselines and baselines:
                return baselines

        # If it wasn't found in a platform directory, return the expected
        # result in the test directory, even if no such file actually exists.
        platform_dir = self.layout_tests_dir()
276
277
        if self.path_exists(self._filesystem.join(platform_dir,
                                                  baseline_filename)):
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
            baselines.append((platform_dir, baseline_filename))

        if baselines:
            return baselines

        return [(None, baseline_filename)]

    def expected_filename(self, filename, suffix):
        """Given a test name, returns an absolute path to its expected results.

        If no expected results are found in any of the searched directories,
        the directory in which the test itself is located will be returned.
        The return value is in the format appropriate for the platform
        (e.g., "\\" for path separators on windows).

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g. '.txt'
            or '.png'.  This should not be None, but may be an empty string.
        platform: the most-specific directory name to use to build the
            search list of directories, e.g., 'chromium-win', or
            'chromium-mac-leopard' (we follow the WebKit format)

        This routine is generic but is implemented here to live alongside
        the other baseline and filename manipulation routines.
        """
        platform_dir, baseline_filename = self.expected_baselines(
            filename, suffix)[0]
        if platform_dir:
307
308
            return self._filesystem.join(platform_dir, baseline_filename)
        return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
309
310
311

    def expected_checksum(self, test):
        """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
312
313
314
315
        path = self.expected_filename(test, '.checksum')
        if not self.path_exists(path):
            return None
        return self._filesystem.read_text_file(path)
316
317
318

    def expected_image(self, test):
        """Returns the image we expect the test to produce."""
319
320
321
322
        path = self.expected_filename(test, '.png')
        if not self.path_exists(path):
            return None
        return self._filesystem.read_binary_file(path)
323
324

    def expected_text(self, test):
325
326
        """Returns the text output we expect the test to produce.
        End-of-line characters are normalized to '\n'."""
327
328
329
330
331
        # FIXME: DRT output is actually utf-8, but since we don't decode the
        # output from DRT (instead treating it as a binary string), we read the
        # baselines as a binary string, too.
        path = self.expected_filename(test, '.txt')
        if not self.path_exists(path):
332
            return ''
333
        text = self._filesystem.read_binary_file(path)
334
        return text.replace("\r\n", "\n")
335

336
    def filename_to_uri(self, filename):
337
        """Convert a test file (which is an absolute path) to a URI."""
338
        LAYOUTTEST_HTTP_DIR = "http/tests/"
339
        LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
340
341
342
343
344

        relative_path = self.relative_test_filename(filename)
        port = None
        use_ssl = False

345
346
        if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
            or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
            relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
            port = 8000

        # Make http/tests/local run as local files. This is to mimic the
        # logic in run-webkit-tests.
        #
        # TODO(dpranke): remove the media reference and the SSL reference?
        if (port and not relative_path.startswith("local/") and
            not relative_path.startswith("media/")):
            if relative_path.startswith("ssl/"):
                port += 443
                protocol = "https"
            else:
                protocol = "http"
            return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)

363
        return path.abspath_to_uri(os.path.abspath(filename))
364

365
366
367
368
369
370
371
372
373
    def tests(self, paths):
        """Return the list of tests found (relative to layout_tests_dir()."""
        return test_files.find(self, paths)

    def test_dirs(self):
        """Returns the list of top-level test directories.

        Used by --clobber-old-results."""
        layout_tests_dir = self.layout_tests_dir()
374
375
        return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
                      self._filesystem.listdir(layout_tests_dir))
376
377

    def path_isdir(self, path):
378
379
380
        """Return True if the path refers to a directory of tests."""
        # Used by test_expectations.py to apply rules to whole directories.
        return self._filesystem.isdir(path)
381
382

    def path_exists(self, path):
383
        """Return True if the path refers to an existing test or baseline."""
384
        # Used by test_expectations.py to determine if an entry refers to a
385
386
        # valid test and by printing.py to determine if baselines exist.
        return self._filesystem.exists(path)
387

388
389
390
391
392
    def driver_cmd_line(self):
        """Prints the DRT command line that will be used."""
        driver = self.create_driver(0)
        return driver.cmd_line()

393
394
395
396
397
398
399
400
401
402
    def update_baseline(self, path, data, encoding):
        """Updates the baseline for a test.

        Args:
            path: the actual path to use for baseline, not the path to
              the test. This function is used to update either generic or
              platform-specific baselines, but we can't infer which here.
            data: contents of the baseline.
            encoding: file encoding to use for the baseline.
        """
403
404
        # FIXME: remove the encoding parameter in favor of text/binary
        # functions.
405
        if encoding is None:
406
407
408
            self._filesystem.write_binary_file(path, data)
        else:
            self._filesystem.write_text_file(path, data)
409
410
411
412
413
414
415
416
417
418
419

    def uri_to_test_name(self, uri):
        """Return the base layout test name for a given URI.

        This returns the test name for a given URI, e.g., if you passed in
        "file:///src/LayoutTests/fast/html/keygen.html" it would return
        "fast/html/keygen.html".

        """
        test = uri
        if uri.startswith("file:///"):
420
            prefix = path.abspath_to_uri(self.layout_tests_dir()) + "/"
421
            return test[len(prefix):]
422
423
424
425
426
427
428
429
430
431
432
433
434
435

        if uri.startswith("http://127.0.0.1:8880/"):
            # websocket tests
            return test.replace('http://127.0.0.1:8880/', '')

        if uri.startswith("http://"):
            # regular HTTP test
            return test.replace('http://127.0.0.1:8000/', 'http/tests/')

        if uri.startswith("https://"):
            return test.replace('https://127.0.0.1:8443/', 'http/tests/')

        raise NotImplementedError('unknown url type: %s' % uri)

436
437
438
439
    def layout_tests_dir(self):
        """Return the absolute path to the top of the LayoutTests directory."""
        return self.path_from_webkit_base('LayoutTests')

440
441
442
443
444
445
446
    def skips_layout_test(self, test_name):
        """Figures out if the givent test is being skipped or not.

        Test categories are handled as well."""
        for test_or_category in self.skipped_layout_tests():
            if test_or_category == test_name:
                return True
447
448
449
450
            category = self._filesystem.join(self.layout_tests_dir(),
                                             test_or_category)
            if (self._filesystem.isdir(category) and
                test_name.startswith(test_or_category)):
451
452
453
                return True
        return False

454
455
    def maybe_make_directory(self, *path):
        """Creates the specified directory if it doesn't already exist."""
456
        self._filesystem.maybe_make_directory(*path)
457
458
459
460
461
462
463
464

    def name(self):
        """Return the name of the port (e.g., 'mac', 'chromium-win-xp').

        Note that this is different from the test_platform_name(), which
        may be different (e.g., 'win-xp' instead of 'chromium-win-xp'."""
        return self._name

465
466
467
468
469
470
471
472
473
474
475
476
    def get_option(self, name, default_value=None):
        # FIXME: Eventually we should not have to do a test for
        # hasattr(), and we should be able to just do
        # self.options.value. See additional FIXME in the constructor.
        if hasattr(self._options, name):
            return getattr(self._options, name)
        return default_value

    def set_option_default(self, name, default_value):
        if not hasattr(self._options, name):
            return setattr(self._options, name, default_value)

477
478
479
    def path_from_webkit_base(self, *comps):
        """Returns the full path to path made by joining the top of the
        WebKit source tree and the list of path components in |*comps|."""
480
        return self._config.path_from_webkit_base(*comps)
481

482
    def script_path(self, script_name):
483
        return self._config.script_path(script_name)
484

485
486
487
488
489
490
    def path_to_test_expectations_file(self):
        """Update the test expectations to the passed-in string.

        This is used by the rebaselining tool. Raises NotImplementedError
        if the port does not use expectations files."""
        raise NotImplementedError('Port.path_to_test_expectations_file')
491

492
493
494
495
    def relative_test_filename(self, filename):
        """Relative unix-style path for a filename under the LayoutTests
        directory. Filenames outside the LayoutTests directory should raise
        an error."""
496
        assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir())
497
498
499
500
        return filename[len(self.layout_tests_dir()) + 1:]

    def results_directory(self):
        """Absolute path to the place to store the test results."""
501
        raise NotImplementedError('Port.results_directory')
502
503

    def setup_test_run(self):
504
        """Perform port-specific work at the beginning of a test run."""
505
506
        pass

507
508
509
510
511
512
    def setup_environ_for_server(self):
        """Perform port-specific work at the beginning of a server launch.

        Returns:
           Operating-system's environment.
        """
513
        return os.environ.copy()
514

515
    def show_results_html_file(self, results_filename):
516
517
        """This routine should display the HTML file pointed at by
        results_filename in a users' browser."""
518
        return self._user.open_url(results_filename)
519

520
    def create_driver(self, worker_number):
521
522
523
        """Return a newly created base.Driver subclass for starting/stopping
        the test driver."""
        raise NotImplementedError('Port.create_driver')
524
525

    def start_helper(self):
526
527
528
529
        """If a port needs to reconfigure graphics settings or do other
        things to ensure a known test configuration, it should override this
        method."""
        pass
530
531
532
533
534

    def start_http_server(self):
        """Start a web server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
535
        if self.get_option('use_apache'):
536
            self._http_server = apache_http_server.LayoutTestApacheHttpd(self,
537
                self.get_option('results_directory'))
538
539
        else:
            self._http_server = http_server.Lighttpd(self,
540
                self.get_option('results_directory'))
541
542
543
544
545
546
547
        self._http_server.start()

    def start_websocket_server(self):
        """Start a websocket server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
        self._websocket_server = websocket_server.PyWebSocket(self,
548
            self.get_option('results_directory'))
549
550
        self._websocket_server.start()

551
552
553
554
    def acquire_http_lock(self):
        self._http_lock = http_lock.HttpLock(None)
        self._http_lock.wait_for_httpd_lock()

555
556
    def stop_helper(self):
        """Shut down the test helper if it is running. Do nothing if
557
558
559
        it isn't, or it isn't available. If a port overrides start_helper()
        it must override this routine as well."""
        pass
560
561
562
563
564
565
566
567
568
569
570
571
572

    def stop_http_server(self):
        """Shut down the http server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._http_server:
            self._http_server.stop()

    def stop_websocket_server(self):
        """Shut down the websocket server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._websocket_server:
            self._websocket_server.stop()

573
574
575
576
    def release_http_lock(self):
        if self._http_lock:
            self._http_lock.cleanup_http_lock()

577
578
579
580
581
582
583
    def test_expectations(self):
        """Returns the test expectations for this port.

        Basically this string should contain the equivalent of a
        test_expectations file. See test_expectations.py for more details."""
        raise NotImplementedError('Port.test_expectations')

584
585
586
587
588
589
590
591
592
    def test_expectations_overrides(self):
        """Returns an optional set of overrides for the test_expectations.

        This is used by ports that have code in two repositories, and where
        it is possible that you might need "downstream" expectations that
        temporarily override the "upstream" expectations until the port can
        sync up the two repos."""
        return None

593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
    def test_base_platform_names(self):
        """Return a list of the 'base' platforms on your port. The base
        platforms represent different architectures, operating systems,
        or implementations (as opposed to different versions of a single
        platform). For example, 'mac' and 'win' might be different base
        platforms, wherease 'mac-tiger' and 'mac-leopard' might be
        different platforms. This routine is used by the rebaselining tool
        and the dashboards, and the strings correspond to the identifiers
        in your test expectations (*not* necessarily the platform names
        themselves)."""
        raise NotImplementedError('Port.base_test_platforms')

    def test_platform_name(self):
        """Returns the string that corresponds to the given platform name
        in the test expectations. This may be the same as name(), or it
        may be different. For example, chromium returns 'mac' for
        'chromium-mac'."""
        raise NotImplementedError('Port.test_platform_name')

    def test_platforms(self):
        """Returns the list of test platform identifiers as used in the
        test_expectations and on dashboards, the rebaselining tool, etc.

        Note that this is not necessarily the same as the list of ports,
        which must be globally unique (e.g., both 'chromium-mac' and 'mac'
        might return 'mac' as a test_platform name'."""
        raise NotImplementedError('Port.platforms')

621
622
623
624
625
626
    def test_platform_name_to_name(self, test_platform_name):
        """Returns the Port platform name that corresponds to the name as
        referenced in the expectations file. E.g., "mac" returns
        "chromium-mac" on the Chromium ports."""
        raise NotImplementedError('Port.test_platform_name_to_name')

627
628
629
630
631
632
633
634
    def version(self):
        """Returns a string indicating the version of a given platform, e.g.
        '-leopard' or '-xp'.

        This is used to help identify the exact port when parsing test
        expectations, determining search paths, and logging information."""
        raise NotImplementedError('Port.version')

635
636
637
638
639
640
641
642
    def test_repository_paths(self):
        """Returns a list of (repository_name, repository_path) tuples
        of its depending code base.  By default it returns a list that only
        contains a ('webkit', <webkitRepossitoryPath>) tuple.
        """
        return [('webkit', self.layout_tests_dir())]


643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
    _WDIFF_DEL = '##WDIFF_DEL##'
    _WDIFF_ADD = '##WDIFF_ADD##'
    _WDIFF_END = '##WDIFF_END##'

    def _format_wdiff_output_as_html(self, wdiff):
        wdiff = cgi.escape(wdiff)
        wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
        wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
        wdiff = wdiff.replace(self._WDIFF_END, "</span>")
        html = "<head><style>.del { background: #faa; } "
        html += ".add { background: #afa; }</style></head>"
        html += "<pre>%s</pre>" % wdiff
        return html

    def _wdiff_command(self, actual_filename, expected_filename):
        executable = self._path_to_wdiff()
        return [executable,
                "--start-delete=%s" % self._WDIFF_DEL,
                "--end-delete=%s" % self._WDIFF_END,
                "--start-insert=%s" % self._WDIFF_ADD,
                "--end-insert=%s" % self._WDIFF_END,
                actual_filename,
                expected_filename]

    @staticmethod
    def _handle_wdiff_error(script_error):
        # Exit 1 means the files differed, any other exit code is an error.
        if script_error.exit_code != 1:
            raise script_error

    def _run_wdiff(self, actual_filename, expected_filename):
        """Runs wdiff and may throw exceptions.
        This is mostly a hook for unit testing."""
        # Diffs are treated as binary as they may include multiple files
        # with conflicting encodings.  Thus we do not decode the output.
        command = self._wdiff_command(actual_filename, expected_filename)
        wdiff = self._executive.run_command(command, decode_output=False,
            error_handler=self._handle_wdiff_error)
        return self._format_wdiff_output_as_html(wdiff)

683
684
685
686
    def wdiff_text(self, actual_filename, expected_filename):
        """Returns a string of HTML indicating the word-level diff of the
        contents of the two filenames. Returns an empty string if word-level
        diffing isn't available."""
687
        if not self._wdiff_available:
688
            return ""
689
        try:
690
691
            # It's possible to raise a ScriptError we pass wdiff invalid paths.
            return self._run_wdiff(actual_filename, expected_filename)
692
        except OSError, e:
693
694
            if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
                # Silently ignore cases where wdiff is missing.
695
                self._wdiff_available = False
696
697
                return ""
            raise
698

699
700
    # This is a class variable so we can test error output easily.
    _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
701

702
    def pretty_patch_text(self, diff_path):
703
        if not self._pretty_patch_available:
704
            return self._pretty_patch_error_html
705
706
        command = ("ruby", "-I", os.path.dirname(self._pretty_patch_path),
                   self._pretty_patch_path, diff_path)
707
        try:
708
709
710
            # Diffs are treated as binary (we pass decode_output=False) as they
            # may contain multiple files of conflicting encodings.
            return self._executive.run_command(command, decode_output=False)
711
        except OSError, e:
712
            # If the system is missing ruby log the error and stop trying.
713
            self._pretty_patch_available = False
714
715
716
            _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
            return self._pretty_patch_error_html
        except ScriptError, e:
717
718
719
720
721
            # If ruby failed to run for some reason, log the command
            # output and stop trying.
            self._pretty_patch_available = False
            _log.error("Failed to run PrettyPatch (%s):\n%s" % (command,
                       e.message_with_output()))
722
            return self._pretty_patch_error_html
723

724
    def default_configuration(self):
725
        return self._config.default_configuration()
726

727
728
729
730
731
732
    #
    # PROTECTED ROUTINES
    #
    # The routines below should only be called by routines in this class
    # or any of its subclasses.
    #
733
734
    def _webkit_build_directory(self, args):
        return self._config.build_directory(args[0])
735
736
737
738
739
740
741
742
743
744
745
746
747

    def _path_to_apache(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache')

    def _path_to_apache_config_file(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache_config_file')

748
    def _path_to_driver(self, configuration=None):
749
        """Returns the full path to the test driver (DumpRenderTree)."""
750
        raise NotImplementedError('Port._path_to_driver')
751
752
753
754

    def _path_to_webcore_library(self):
        """Returns the full path to a built copy of WebCore."""
        raise NotImplementedError('Port.path_to_webcore_library')
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805

    def _path_to_helper(self):
        """Returns the full path to the layout_test_helper binary, which
        is used to help configure the system for the test run, or None
        if no helper is needed.

        This is likely only used by start/stop_helper()."""
        raise NotImplementedError('Port._path_to_helper')

    def _path_to_image_diff(self):
        """Returns the full path to the image_diff binary, or None if it
        is not available.

        This is likely used only by diff_image()"""
        raise NotImplementedError('Port.path_to_image_diff')

    def _path_to_lighttpd(self):
        """Returns the path to the LigHTTPd binary.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd')

    def _path_to_lighttpd_modules(self):
        """Returns the path to the LigHTTPd modules directory.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_modules')

    def _path_to_lighttpd_php(self):
        """Returns the path to the LigHTTPd PHP executable.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_php')

    def _path_to_wdiff(self):
        """Returns the full path to the wdiff binary, or None if it is
        not available.

        This is likely used only by wdiff_text()"""
        raise NotImplementedError('Port._path_to_wdiff')

    def _shut_down_http_server(self, pid):
        """Forcefully and synchronously kills the web server.

        This routine should only be called from http_server.py or its
        subclasses."""
        raise NotImplementedError('Port._shut_down_http_server')

    def _webkit_baseline_path(self, platform):
        """Return the  full path to the top of the baseline tree for a
        given platform."""
806
807
        return self._filesystem.join(self.layout_tests_dir(), 'platform',
                                     platform)
808
809
810
811
812


class Driver:
    """Abstract interface for the DumpRenderTree interface."""

813
    def __init__(self, port, worker_number):
814
815
816
817
818
819
        """Initialize a Driver to subsequently run tests.

        Typically this routine will spawn DumpRenderTree in a config
        ready for subsequent input.

        port - reference back to the port object.
820
        worker_number - identifier for a particular worker/driver instance
821
        """
822
823
        raise NotImplementedError('Driver.__init__')

824
    def run_test(self, test_input):
825
826
827
828
829
830
        """Run a single test and return the results.

        Note that it is okay if a test times out or crashes and leaves
        the driver in an indeterminate state. The upper layers of the program
        are responsible for cleaning up and ensuring things are okay.

831
832
        Args:
          test_input: a TestInput object
833

834
835
        Returns a TestOutput object.
        """
836
837
        raise NotImplementedError('Driver.run_test')

838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
    # FIXME: This is static so we can test it w/o creating a Base instance.
    @classmethod
    def _command_wrapper(cls, wrapper_option):
        # Hook for injecting valgrind or other runtime instrumentation,
        # used by e.g. tools/valgrind/valgrind_tests.py.
        wrapper = []
        browser_wrapper = os.environ.get("BROWSER_WRAPPER", None)
        if browser_wrapper:
            # FIXME: There seems to be no reason to use BROWSER_WRAPPER over --wrapper.
            # Remove this code any time after the date listed below.
            _log.error("BROWSER_WRAPPER is deprecated, please use --wrapper instead.")
            _log.error("BROWSER_WRAPPER will be removed any time after June 1st 2010 and your scripts will break.")
            wrapper += [browser_wrapper]

        if wrapper_option:
            wrapper += shlex.split(wrapper_option)
        return wrapper

856
857
858
859
860
861
862
    def poll(self):
        """Returns None if the Driver is still running. Returns the returncode
        if it has exited."""
        raise NotImplementedError('Driver.poll')

    def stop(self):
        raise NotImplementedError('Driver.stop')