|
1 #!/usr/bin/perl -w |
|
2 |
|
3 # Copyright (C) 2007, 2008 Apple Inc. All rights reserved. |
|
4 # |
|
5 # Redistribution and use in source and binary forms, with or without |
|
6 # modification, are permitted provided that the following conditions |
|
7 # are met: |
|
8 # |
|
9 # 1. Redistributions of source code must retain the above copyright |
|
10 # notice, this list of conditions and the following disclaimer. |
|
11 # 2. Redistributions in binary form must reproduce the above copyright |
|
12 # notice, this list of conditions and the following disclaimer in the |
|
13 # documentation and/or other materials provided with the distribution. |
|
14 # 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of |
|
15 # its contributors may be used to endorse or promote products derived |
|
16 # from this software without specific prior written permission. |
|
17 # |
|
18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY |
|
19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
|
21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY |
|
22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
|
23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
|
24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
|
25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
|
27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
28 |
|
29 # This script attempts to find the point at which a regression (or progression) |
|
30 # of behavior occurred by searching WebKit nightly builds. |
|
31 |
|
32 # To override the location where the nightly builds are downloaded or the path |
|
33 # to the Safari web browser, create a ~/.bisect-buildsrc file with one or more of |
|
34 # the following lines (use "~/" to specify a path from your home directory): |
|
35 # |
|
36 # $branch = "branch-name"; |
|
37 # $nightlyDownloadDirectory = "~/path/to/nightly/downloads"; |
|
38 # $safariPath = "/path/to/Safari.app"; |
|
39 |
|
40 use strict; |
|
41 |
|
42 use File::Basename; |
|
43 use File::Path; |
|
44 use File::Spec; |
|
45 use File::Temp qw(tempfile); |
|
46 use Getopt::Long; |
|
47 use Time::HiRes qw(usleep); |
|
48 |
|
49 sub createTempFile($); |
|
50 sub downloadNightly($$$); |
|
51 sub findMacOSXVersion(); |
|
52 sub findNearestNightlyIndex(\@$$); |
|
53 sub findSafariVersion($); |
|
54 sub loadSettings(); |
|
55 sub makeNightlyList($$$$); |
|
56 sub max($$) { return $_[0] > $_[1] ? $_[0] : $_[1]; } |
|
57 sub mountAndRunNightly($$$$); |
|
58 sub parseRevisions($$;$); |
|
59 sub printStatus($$$); |
|
60 sub promptForTest($); |
|
61 |
|
62 loadSettings(); |
|
63 |
|
64 my %validBranches = map { $_ => 1 } qw(feature-branch trunk); |
|
65 my $branch = $Settings::branch; |
|
66 my $nightlyDownloadDirectory = $Settings::nightlyDownloadDirectory; |
|
67 my $safariPath = $Settings::safariPath; |
|
68 |
|
69 my @nightlies; |
|
70 |
|
71 my $isProgression; |
|
72 my $localOnly; |
|
73 my @revisions; |
|
74 my $sanityCheck; |
|
75 my $showHelp; |
|
76 my $testURL; |
|
77 |
|
78 # Fix up -r switches in @ARGV |
|
79 @ARGV = map { /^(-r)(.+)$/ ? ($1, $2) : $_ } @ARGV; |
|
80 |
|
81 my $result = GetOptions( |
|
82 "b|branch=s" => \$branch, |
|
83 "d|download-directory=s" => \$nightlyDownloadDirectory, |
|
84 "h|help" => \$showHelp, |
|
85 "l|local!" => \$localOnly, |
|
86 "p|progression!" => \$isProgression, |
|
87 "r|revisions=s" => \&parseRevisions, |
|
88 "safari-path=s" => \$safariPath, |
|
89 "s|sanity-check!" => \$sanityCheck, |
|
90 ); |
|
91 $testURL = shift @ARGV; |
|
92 |
|
93 $branch = "feature-branch" if $branch eq "feature"; |
|
94 if (!exists $validBranches{$branch}) { |
|
95 print STDERR "ERROR: Invalid branch '$branch'\n"; |
|
96 $showHelp = 1; |
|
97 } |
|
98 |
|
99 if (!$result || $showHelp || scalar(@ARGV) > 0) { |
|
100 print STDERR "Search WebKit nightly builds for changes in behavior.\n"; |
|
101 print STDERR "Usage: " . basename($0) . " [options] [url]\n"; |
|
102 print STDERR <<END; |
|
103 [-b|--branch name] name of the nightly build branch (default: trunk) |
|
104 [-d|--download-directory dir] nightly build download directory (default: ~/Library/Caches/WebKit-Nightlies) |
|
105 [-h|--help] show this help message |
|
106 [-l|--local] only use local (already downloaded) nightlies |
|
107 [-p|--progression] searching for a progression, not a regression |
|
108 [-r|--revision M[:N]] specify starting (and optional ending) revisions to search |
|
109 [--safari-path path] path to Safari application bundle (default: /Applications/Safari.app) |
|
110 [-s|--sanity-check] verify both starting and ending revisions before bisecting |
|
111 END |
|
112 exit 1; |
|
113 } |
|
114 |
|
115 my $nightlyWebSite = "http://nightly.webkit.org"; |
|
116 my $nightlyBuildsURLBase = $nightlyWebSite . File::Spec->catdir("/builds", $branch, "mac"); |
|
117 my $nightlyFilesURLBase = $nightlyWebSite . File::Spec->catdir("/files", $branch, "mac"); |
|
118 |
|
119 $nightlyDownloadDirectory = glob($nightlyDownloadDirectory) if $nightlyDownloadDirectory =~ /^~/; |
|
120 $safariPath = glob($safariPath) if $safariPath =~ /^~/; |
|
121 $safariPath = File::Spec->catdir($safariPath, "Contents/MacOS/Safari") if $safariPath =~ m#\.app/*#; |
|
122 |
|
123 $nightlyDownloadDirectory = File::Spec->catdir($nightlyDownloadDirectory, $branch); |
|
124 if (! -d $nightlyDownloadDirectory) { |
|
125 mkpath($nightlyDownloadDirectory, 0, 0755) || die "Could not create $nightlyDownloadDirectory: $!"; |
|
126 } |
|
127 |
|
128 @nightlies = makeNightlyList($localOnly, $nightlyDownloadDirectory, findMacOSXVersion(), findSafariVersion($safariPath)); |
|
129 |
|
130 my $startIndex = $revisions[0] ? findNearestNightlyIndex(@nightlies, $revisions[0], 'ceil') : 0; |
|
131 my $endIndex = $revisions[1] ? findNearestNightlyIndex(@nightlies, $revisions[1], 'floor') : $#nightlies; |
|
132 |
|
133 my $tempFile = createTempFile($testURL); |
|
134 |
|
135 if ($sanityCheck) { |
|
136 my $didReproduceBug; |
|
137 |
|
138 do { |
|
139 printf "\nChecking starting revision r%s...\n", |
|
140 $nightlies[$startIndex]->{rev}; |
|
141 downloadNightly($nightlies[$startIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory); |
|
142 mountAndRunNightly($nightlies[$startIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile); |
|
143 $didReproduceBug = promptForTest($nightlies[$startIndex]->{rev}); |
|
144 $startIndex-- if $didReproduceBug < 0; |
|
145 } while ($didReproduceBug < 0); |
|
146 die "ERROR: Bug reproduced in starting revision! Do you need to test an earlier revision or for a progression?" |
|
147 if $didReproduceBug && !$isProgression; |
|
148 die "ERROR: Bug not reproduced in starting revision! Do you need to test an earlier revision or for a regression?" |
|
149 if !$didReproduceBug && $isProgression; |
|
150 |
|
151 do { |
|
152 printf "\nChecking ending revision r%s...\n", |
|
153 $nightlies[$endIndex]->{rev}; |
|
154 downloadNightly($nightlies[$endIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory); |
|
155 mountAndRunNightly($nightlies[$endIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile); |
|
156 $didReproduceBug = promptForTest($nightlies[$endIndex]->{rev}); |
|
157 $endIndex++ if $didReproduceBug < 0; |
|
158 } while ($didReproduceBug < 0); |
|
159 die "ERROR: Bug NOT reproduced in ending revision! Do you need to test a later revision or for a progression?" |
|
160 if !$didReproduceBug && !$isProgression; |
|
161 die "ERROR: Bug reproduced in ending revision! Do you need to test a later revision or for a regression?" |
|
162 if $didReproduceBug && $isProgression; |
|
163 } |
|
164 |
|
165 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression); |
|
166 |
|
167 my %brokenRevisions = (); |
|
168 while (abs($endIndex - $startIndex) > 1) { |
|
169 my $index = $startIndex + int(($endIndex - $startIndex) / 2); |
|
170 |
|
171 my $didReproduceBug; |
|
172 do { |
|
173 if (exists $nightlies[$index]) { |
|
174 my $buildsLeft = max(max(0, $endIndex - $index - 1), max(0, $index - $startIndex - 1)); |
|
175 my $plural = $buildsLeft == 1 ? "" : "s"; |
|
176 printf "\nChecking revision r%s (%d build%s left to test after this)...\n", $nightlies[$index]->{rev}, $buildsLeft, $plural; |
|
177 downloadNightly($nightlies[$index]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory); |
|
178 mountAndRunNightly($nightlies[$index]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile); |
|
179 $didReproduceBug = promptForTest($nightlies[$index]->{rev}); |
|
180 } |
|
181 if ($didReproduceBug < 0) { |
|
182 $brokenRevisions{$nightlies[$index]->{rev}} = $nightlies[$index]->{file}; |
|
183 delete $nightlies[$index]; |
|
184 $endIndex--; |
|
185 $index = $startIndex + int(($endIndex - $startIndex) / 2); |
|
186 } |
|
187 } while ($didReproduceBug < 0); |
|
188 |
|
189 if ($didReproduceBug && !$isProgression || !$didReproduceBug && $isProgression) { |
|
190 $endIndex = $index; |
|
191 } else { |
|
192 $startIndex = $index; |
|
193 } |
|
194 |
|
195 print "\nBroken revisions skipped: r" . join(", r", keys %brokenRevisions) . "\n" |
|
196 if scalar keys %brokenRevisions > 0; |
|
197 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression); |
|
198 } |
|
199 |
|
200 unlink $tempFile if $tempFile; |
|
201 |
|
202 exit 0; |
|
203 |
|
204 sub createTempFile($) |
|
205 { |
|
206 my ($url) = @_; |
|
207 |
|
208 return undef if !$url; |
|
209 |
|
210 my ($fh, $tempFile) = tempfile( |
|
211 basename($0) . "-XXXXXXXX", |
|
212 DIR => File::Spec->tmpdir(), |
|
213 SUFFIX => ".html", |
|
214 UNLINK => 0, |
|
215 ); |
|
216 print $fh "<meta http-equiv=\"refresh\" content=\"0; $url\">\n"; |
|
217 close($fh); |
|
218 |
|
219 return $tempFile; |
|
220 } |
|
221 |
|
222 sub downloadNightly($$$) |
|
223 { |
|
224 my ($filename, $urlBase, $directory) = @_; |
|
225 my $path = File::Spec->catfile($directory, $filename); |
|
226 if (! -f $path) { |
|
227 print "Downloading $filename to $directory...\n"; |
|
228 `curl -# -o '$path' '$urlBase/$filename'`; |
|
229 } |
|
230 } |
|
231 |
|
232 sub findMacOSXVersion() |
|
233 { |
|
234 my $version; |
|
235 open(SW_VERS, "-|", "/usr/bin/sw_vers") || die; |
|
236 while (<SW_VERS>) { |
|
237 $version = $1 if /^ProductVersion:\s+([^\s]+)/; |
|
238 } |
|
239 close(SW_VERS); |
|
240 return $version; |
|
241 } |
|
242 |
|
243 sub findNearestNightlyIndex(\@$$) |
|
244 { |
|
245 my ($nightlies, $revision, $round) = @_; |
|
246 |
|
247 my $lowIndex = 0; |
|
248 my $highIndex = $#{$nightlies}; |
|
249 |
|
250 return $highIndex if uc($revision) eq 'HEAD' || $revision >= $nightlies->[$highIndex]->{rev}; |
|
251 return $lowIndex if $revision <= $nightlies->[$lowIndex]->{rev}; |
|
252 |
|
253 while (abs($highIndex - $lowIndex) > 1) { |
|
254 my $index = $lowIndex + int(($highIndex - $lowIndex) / 2); |
|
255 if ($revision < $nightlies->[$index]->{rev}) { |
|
256 $highIndex = $index; |
|
257 } elsif ($revision > $nightlies->[$index]->{rev}) { |
|
258 $lowIndex = $index; |
|
259 } else { |
|
260 return $index; |
|
261 } |
|
262 } |
|
263 |
|
264 return ($round eq "floor") ? $lowIndex : $highIndex; |
|
265 } |
|
266 |
|
267 sub findSafariVersion($) |
|
268 { |
|
269 my ($path) = @_; |
|
270 my $versionPlist = File::Spec->catdir(dirname(dirname($path)), "version.plist"); |
|
271 my $version; |
|
272 open(PLIST, "< $versionPlist") || die; |
|
273 while (<PLIST>) { |
|
274 if (m#^\s*<key>CFBundleShortVersionString</key>#) { |
|
275 $version = <PLIST>; |
|
276 $version =~ s#^\s*<string>([0-9.]+)[^<]*</string>\s*[\r\n]*#$1#; |
|
277 } |
|
278 } |
|
279 close(PLIST); |
|
280 return $version; |
|
281 } |
|
282 |
|
283 sub loadSettings() |
|
284 { |
|
285 package Settings; |
|
286 |
|
287 our $branch = "trunk"; |
|
288 our $nightlyDownloadDirectory = File::Spec->catdir($ENV{HOME}, "Library/Caches/WebKit-Nightlies"); |
|
289 our $safariPath = "/Applications/Safari.app"; |
|
290 |
|
291 my $rcfile = File::Spec->catdir($ENV{HOME}, ".bisect-buildsrc"); |
|
292 return if !-f $rcfile; |
|
293 |
|
294 my $result = do $rcfile; |
|
295 die "Could not parse $rcfile: $@" if $@; |
|
296 } |
|
297 |
|
298 sub makeNightlyList($$$$) |
|
299 { |
|
300 my ($useLocalFiles, $localDirectory, $macOSXVersion, $safariVersion) = @_; |
|
301 my @files; |
|
302 |
|
303 if ($useLocalFiles) { |
|
304 opendir(DIR, $localDirectory) || die "$!"; |
|
305 foreach my $file (readdir(DIR)) { |
|
306 if ($file =~ /^WebKit-SVN-r([0-9]+)\.dmg$/) { |
|
307 push(@files, +{ rev => $1, file => $file }); |
|
308 } |
|
309 } |
|
310 closedir(DIR); |
|
311 } else { |
|
312 open(NIGHTLIES, "curl -s $nightlyBuildsURLBase/all |") || die; |
|
313 |
|
314 while (my $line = <NIGHTLIES>) { |
|
315 chomp $line; |
|
316 my ($revision, $timestamp, $url) = split(/,/, $line); |
|
317 my $nightly = basename($url); |
|
318 push(@files, +{ rev => $revision, file => $nightly }); |
|
319 } |
|
320 close(NIGHTLIES); |
|
321 } |
|
322 |
|
323 if (eval "v$macOSXVersion" ge v10.5) { |
|
324 if ($safariVersion eq "4 Public Beta") { |
|
325 @files = grep { $_->{rev} >= 39682 } @files; |
|
326 } elsif (eval "v$safariVersion" ge v3.2) { |
|
327 @files = grep { $_->{rev} >= 37348 } @files; |
|
328 } elsif (eval "v$safariVersion" ge v3.1) { |
|
329 @files = grep { $_->{rev} >= 29711 } @files; |
|
330 } elsif (eval "v$safariVersion" ge v3.0) { |
|
331 @files = grep { $_->{rev} >= 25124 } @files; |
|
332 } elsif (eval "v$safariVersion" ge v2.0) { |
|
333 @files = grep { $_->{rev} >= 19594 } @files; |
|
334 } else { |
|
335 die "Requires Safari 2.0 or newer"; |
|
336 } |
|
337 } elsif (eval "v$macOSXVersion" ge v10.4) { |
|
338 if ($safariVersion eq "4 Public Beta") { |
|
339 @files = grep { $_->{rev} >= 39682 } @files; |
|
340 } elsif (eval "v$safariVersion" ge v3.2) { |
|
341 @files = grep { $_->{rev} >= 37348 } @files; |
|
342 } elsif (eval "v$safariVersion" ge v3.1) { |
|
343 @files = grep { $_->{rev} >= 29711 } @files; |
|
344 } elsif (eval "v$safariVersion" ge v3.0) { |
|
345 @files = grep { $_->{rev} >= 19992 } @files; |
|
346 } elsif (eval "v$safariVersion" ge v2.0) { |
|
347 @files = grep { $_->{rev} >= 11976 } @files; |
|
348 } else { |
|
349 die "Requires Safari 2.0 or newer"; |
|
350 } |
|
351 } else { |
|
352 die "Requires Mac OS X 10.4 (Tiger) or 10.5 (Leopard)"; |
|
353 } |
|
354 |
|
355 my $nightlycmp = sub { return $a->{rev} <=> $b->{rev}; }; |
|
356 |
|
357 return sort $nightlycmp @files; |
|
358 } |
|
359 |
|
360 sub mountAndRunNightly($$$$) |
|
361 { |
|
362 my ($filename, $directory, $safari, $tempFile) = @_; |
|
363 my $mountPath = "/Volumes/WebKit"; |
|
364 my $webkitApp = File::Spec->catfile($mountPath, "WebKit.app"); |
|
365 my $diskImage = File::Spec->catfile($directory, $filename); |
|
366 my $devNull = File::Spec->devnull(); |
|
367 |
|
368 my $i = 0; |
|
369 while (-e $mountPath) { |
|
370 $i++; |
|
371 usleep 100 if $i > 1; |
|
372 `hdiutil detach '$mountPath' 2> $devNull`; |
|
373 die "Could not unmount $diskImage at $mountPath" if $i > 100; |
|
374 } |
|
375 die "Can't mount $diskImage: $mountPath already exists!" if -e $mountPath; |
|
376 |
|
377 print "Mounting disk image and running WebKit...\n"; |
|
378 `hdiutil attach '$diskImage'`; |
|
379 $i = 0; |
|
380 while (! -e $webkitApp) { |
|
381 usleep 100; |
|
382 $i++; |
|
383 die "Could not mount $diskImage at $mountPath" if $i > 100; |
|
384 } |
|
385 |
|
386 my $frameworkPath; |
|
387 if (-d "/Volumes/WebKit/WebKit.app/Contents/Frameworks") { |
|
388 my $osXVersion = join('.', (split(/\./, findMacOSXVersion()))[0..1]); |
|
389 $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Frameworks/$osXVersion"; |
|
390 } else { |
|
391 $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Resources"; |
|
392 } |
|
393 |
|
394 $tempFile ||= ""; |
|
395 `DYLD_FRAMEWORK_PATH=$frameworkPath WEBKIT_UNSET_DYLD_FRAMEWORK_PATH=YES $safari $tempFile`; |
|
396 |
|
397 `hdiutil detach '$mountPath' 2> $devNull`; |
|
398 } |
|
399 |
|
400 sub parseRevisions($$;$) |
|
401 { |
|
402 my ($optionName, $value, $ignored) = @_; |
|
403 |
|
404 if ($value =~ /^r?([0-9]+|HEAD):?$/i) { |
|
405 push(@revisions, $1); |
|
406 die "Too many revision arguments specified" if scalar @revisions > 2; |
|
407 } elsif ($value =~ /^r?([0-9]+):?r?([0-9]+|HEAD)$/i) { |
|
408 $revisions[0] = $1; |
|
409 $revisions[1] = $2; |
|
410 } else { |
|
411 die "Unknown revision '$value': expected 'M' or 'M:N'"; |
|
412 } |
|
413 } |
|
414 |
|
415 sub printStatus($$$) |
|
416 { |
|
417 my ($startRevision, $endRevision, $isProgression) = @_; |
|
418 printf "\n%s: r%s %s: r%s\n", |
|
419 $isProgression ? "Fails" : "Works", $startRevision, |
|
420 $isProgression ? "Works" : "Fails", $endRevision; |
|
421 } |
|
422 |
|
423 sub promptForTest($) |
|
424 { |
|
425 my ($revision) = @_; |
|
426 print "Did the bug reproduce in r$revision (yes/no/broken)? "; |
|
427 my $answer = <STDIN>; |
|
428 return 1 if $answer =~ /^(1|y.*)$/i; |
|
429 return -1 if $answer =~ /^(-1|b.*)$/i; # Broken |
|
430 return 0; |
|
431 } |
|
432 |