Commit 25d78b5f authored by ddkilzer's avatar ddkilzer
Browse files

WebKitTools:

        Reviewed by Timothy Hatcher.

        - http://bugzilla.opendarwin.org/show_bug.cgi?id=9794
          Teach run-webkit-tests how to ignore tests with performance improvements

        * Scripts/run-webkit-tests: Speed up test list generation, implement --ignore-tests
        feature, and minor clean up.



git-svn-id: http://svn.webkit.org/repository/webkit/trunk@15248 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent e6e3e565
2006-07-09 David Kilzer <ddkilzer@kilzer.net>
Reviewed by Timothy Hatcher.
- http://bugzilla.opendarwin.org/show_bug.cgi?id=9794
Teach run-webkit-tests how to ignore tests with performance improvements
* Scripts/run-webkit-tests: Speed up test list generation, implement --ignore-tests
feature, and minor clean up.
2006-07-08 Darin Adler <darin@apple.com>
Reviewed by Geoff (well, half of it at least).
......
......@@ -48,6 +48,7 @@ use warnings;
use Cwd;
use File::Basename;
use File::Copy;
use File::Find;
use File::Path;
use File::Spec;
use File::Spec::Functions;
......@@ -67,6 +68,7 @@ sub numericcmp($$);
sub openDumpRenderTreeIfNeeded();
sub openHTTPDIfNeeded();
sub pathcmp($$);
sub processIgnoreTests($);
sub slowestcmp($$);
sub splitpath($);
......@@ -74,6 +76,7 @@ sub splitpath($);
my $checkLeaks = '';
my $guardMalloc = '';
my $httpdPort = 8000;
my $ignoreTests = '';
my $launchSafari = 1;
my $pixelTests = '';
my $quiet = '';
......@@ -81,6 +84,7 @@ my $repaintSweepHorizontally = '';
my $repaintTests = '';
my $report10Slowest = 0;
my $resetResults = 0;
my $showHelp = 0;
my $singly = 0;
my $testHTTP = 1;
my $testOnlySVGs = '';
......@@ -100,10 +104,37 @@ if (isCygwin()) {
$actualTag = "diffs-win";
}
GetOptions(
my $usage =
"Usage: " . basename($0) . " [options] [testdir|testpath ...]\n" .
" -g|--guard-malloc Enable malloc guard\n" .
" --help Show this help message\n" .
" -h|--horizontal-sweep Change repaint to sweep horizontally instead of vertically (implies --repaint-tests)\n" .
" --[no-]http Run (or do not run) http tests (default: " . ($testHTTP ? "run" : "do not run") . ")\n" .
" -i|--ignore-tests Comma-separated list of directories or tests to ignore\n" .
" --[no-]launch-safari Launch (or do not launch) Safari to display test results (default: "
. ($launchSafari ? "launch" : "do not launch") . ")\n" .
" -l|--leaks Enable leaks checking\n" .
" -p|--pixel-tests Enable pixel tests\n" .
" --port Web server port to use with http tests\n" .
" -q|--quiet Less verbose output\n" .
" -r|--repaint-tests Run repaint tests (implies --pixel-tests)\n" .
" --reset-results Reset ALL results (including pixel tests if --pixel-tests is set)\n" .
" -o|--results-directory Output results directory (default: " . $testResultsDirectory . ")\n" .
" -1|--singly Isolate each test case run (implies --verbose)\n" .
" --slowest Report the 10 slowest tests\n" .
" --svg Run only SVG tests (implies --pixel-tests)\n" .
" -v|--verbose More verbose output (overrides --quiet)\n" .
" --debug|--release Set DumpRenderTree build configuration\n";
# Parse out build options (--debug or --release) first
my $buildConfiguration = setConfiguration();
my $getOptionsResult = GetOptions(
'guard-malloc|g' => \$guardMalloc,
'help' => \$showHelp,
'horizontal-sweep|h' => \$repaintSweepHorizontally,
'http!' => \$testHTTP,
'ignore-tests|i=s' => \$ignoreTests,
'launch-safari!' => \$launchSafari,
'leaks|l' => \$checkLeaks,
'pixel-tests|p' => \$pixelTests,
......@@ -118,17 +149,27 @@ GetOptions(
'verbose|v' => \$verbose,
);
if (!$getOptionsResult || $showHelp) {
print STDERR $usage;
exit 1;
}
$repaintTests = 1 if $repaintSweepHorizontally;
$pixelTests = 1 if $testOnlySVGs;
$pixelTests = 1 if $repaintTests;
setConfiguration();
$verbose = 1 if $singly;
my $productDir = productDir();
chdirWebKit();
my $buildResult = system "WebKitTools/Scripts/build-dumprendertree", @ARGV;
my @buildOptions = ();
if ($buildConfiguration) {
push @buildOptions, '--' . lc($buildConfiguration);
}
my $buildResult = system "WebKitTools/Scripts/build-dumprendertree", @buildOptions;
exit WEXITSTATUS($buildResult) if $buildResult;
my $dumpToolName = "DumpRenderTree";
......@@ -140,53 +181,75 @@ die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTe
checkFrameworks();
my $layoutTestsName = $testOnlySVGs? "LayoutTests/svg" : "LayoutTests";
my $testDirectory = getcwd() . "/$layoutTestsName";
my $testResults = "$testResultsDirectory/results.html";
my $layoutTestsName = $testOnlySVGs ? "LayoutTests/svg" : "LayoutTests";
my $testDirectory = File::Spec->rel2abs($layoutTestsName);
my $testResults = catfile($testResultsDirectory, "results.html");
print "Running tests from $testDirectory\n";
my @tests = ();
my $prunePart = "\\( -name resources \\! -prune \\)";
my $extensionPart = "-name '*.html' -or -name '*.shtml' -or -name '*.xml' -or -name '*.xhtml' -or -name '*.pl' -or -name '*.php'";
my %ignoredFiles = ();
my %ignoredDirectories = ();
my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources);
my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml pl php);
if ($testOnlySVGs) {
$extensionPart = "-name '*.svg' -or -name '*.xml'";
%supportedFileExtensions = map { $_ => 1 } qw(svg xml);
} elsif (checkWebCoreSVGSupport($testOnlySVGs)) {
$extensionPart .= " -or -name '*.svg'";
$supportedFileExtensions{'svg'} = 1;
} else {
$prunePart .= " -or \\( -name svg \\! -prune \\)";
$ignoredLocalDirectories{'svg'} = 1;
}
if (!$testHTTP) {
$prunePart .= " -or \\( -name http \\! -prune \\)";
$ignoredDirectories{'http'} = 1;
}
my $findArguments = "$prunePart -or $extensionPart";
my $foundTestName = 0;
if ($ignoreTests) {
processIgnoreTests($ignoreTests);
}
my $directoryFilter = sub {
return () if exists $ignoredLocalDirectories{basename($File::Find::dir)};
return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)};
return @_;
};
my $fileFilter = sub {
my $filename = $_;
if ($filename =~ /\.([^.]+)$/) {
if (exists $supportedFileExtensions{$1}) {
my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory);
push @tests, $path if !exists $ignoredFiles{$path};
}
}
};
for my $test (@ARGV) {
next if $test =~ /^-/;
$foundTestName = 1;
$test =~ s/^$testDirectory\///;
if ($test =~ /^\//) {
print "can't run test outside $testDirectory\n";
} elsif (-f "$testDirectory/$test") {
if ($test !~ /\.(html|shtml|xml|xhtml|pl|php|svg)$/) {
$test =~ s/^($layoutTestsName|$testDirectory)\///;
my $fullPath = catfile($testDirectory, $test);
if (file_name_is_absolute($test)) {
print "can't run test $test outside $testDirectory\n";
} elsif (-f $fullPath) {
my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$});
if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) {
print "test $test does not have a supported extension\n";
} elsif ($testHTTP || $test !~ /^http\//) {
} elsif ($testHTTP || $pathname !~ /^http\//) {
push @tests, $test;
}
} elsif (-d "$testDirectory/$test") {
push @tests, sort pathcmp map { chomp; s-^$testDirectory/--; $_; } `find -L "$testDirectory/$test" $findArguments`;
} elsif (-d $fullPath) {
find({ preprocess => $directoryFilter, wanted => $fileFilter }, $fullPath);
} else {
print "test $test not found\n";
}
}
if (!$foundTestName) {
@tests = sort pathcmp map { chomp; s-^$testDirectory/--; $_; } `find -L "$testDirectory" $findArguments`;
if (!scalar @ARGV) {
find({ preprocess => $directoryFilter, wanted => $fileFilter }, $testDirectory);
}
die "no tests to run\n" if !@tests;
@tests = sort pathcmp @tests;
my %counts;
my %tests;
my %imagesPresent;
......@@ -231,7 +294,7 @@ for my $test (@tests) {
my $base = $test;
$base =~ s/\.[a-zA-Z]+$//;
if ($verbose || $singly) {
if ($verbose) {
print "running $test -> ";
$atLineStart = 0;
} elsif (!$quiet) {
......@@ -349,7 +412,7 @@ for my $test (@tests) {
}
if (!defined $expected) {
if ($verbose || $singly) {
if ($verbose) {
print "new " . ($resetResults ? "result" : "test") ."\n";
$atLineStart = 1;
}
......@@ -360,7 +423,7 @@ for my $test (@tests) {
unlink "$testResultsDirectory/$base-$actualTag.txt";
unlink "$testResultsDirectory/$base-$diffsTag.txt";
} elsif ($actual eq $expected && $diffResult eq "passed") {
if ($verbose || $singly) {
if ($verbose) {
print "succeeded\n";
$atLineStart = 1;
}
......@@ -368,7 +431,7 @@ for my $test (@tests) {
unlink "$testResultsDirectory/$base-$actualTag.txt";
unlink "$testResultsDirectory/$base-$diffsTag.txt";
} else {
unless ($verbose || $singly) {
unless ($verbose) {
print "\n" unless $atLineStart;
print "$test -> ";
}
......@@ -814,3 +877,25 @@ sub fileNameWithNumber($$)
return "$base$number" if ($number > 1);
return $base;
}
sub processIgnoreTests($) {
my @ignoreList = split(/\s*,\s*/, shift);
my $addIgnoredDirectories = sub {
return () if exists $ignoredLocalDirectories{basename($File::Find::dir)};
$ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)} = 1;
return @_;
};
foreach my $item (@ignoreList) {
my $path = catfile($testDirectory, $item);
if (-d $path) {
$ignoredDirectories{$item} = 1;
find({ preprocess => $addIgnoredDirectories, wanted => sub {} }, $path);
}
elsif (-f $path) {
$ignoredFiles{$item} = 1;
}
else {
print "ignoring '$item' on ignore-tests list\n";
}
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment