|
1 # Copyright (C) 2009 Google Inc. All rights reserved. |
|
2 # |
|
3 # Redistribution and use in source and binary forms, with or without |
|
4 # modification, are permitted provided that the following conditions are |
|
5 # met: |
|
6 # |
|
7 # * Redistributions of source code must retain the above copyright |
|
8 # notice, this list of conditions and the following disclaimer. |
|
9 # * Redistributions in binary form must reproduce the above |
|
10 # copyright notice, this list of conditions and the following disclaimer |
|
11 # in the documentation and/or other materials provided with the |
|
12 # distribution. |
|
13 # * Neither the name of Google Inc. nor the names of its |
|
14 # contributors may be used to endorse or promote products derived from |
|
15 # this software without specific prior written permission. |
|
16 # |
|
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
28 |
|
29 import unittest |
|
30 |
|
31 from webkitpy.common.net.buildbot import BuildBot, Builder, Build, LayoutTestResults |
|
32 |
|
33 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup |
|
34 |
|
35 |
|
36 class BuilderTest(unittest.TestCase): |
|
37 def _install_fetch_build(self, failure): |
|
38 def _mock_fetch_build(build_number): |
|
39 build = Build( |
|
40 builder=self.builder, |
|
41 build_number=build_number, |
|
42 revision=build_number + 1000, |
|
43 is_green=build_number < 4 |
|
44 ) |
|
45 build._layout_test_results = LayoutTestResults( |
|
46 "http://buildbot.example.com/foo", { |
|
47 LayoutTestResults.fail_key: failure(build_number), |
|
48 }) |
|
49 return build |
|
50 self.builder._fetch_build = _mock_fetch_build |
|
51 |
|
52 def setUp(self): |
|
53 self.buildbot = BuildBot() |
|
54 self.builder = Builder(u"Test Builder \u2661", self.buildbot) |
|
55 self._install_fetch_build(lambda build_number: ["test1", "test2"]) |
|
56 |
|
57 def test_find_failure_transition(self): |
|
58 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
59 self.assertEqual(green_build.revision(), 1003) |
|
60 self.assertEqual(red_build.revision(), 1004) |
|
61 |
|
62 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10), look_back_limit=2) |
|
63 self.assertEqual(green_build, None) |
|
64 self.assertEqual(red_build.revision(), 1008) |
|
65 |
|
66 def test_none_build(self): |
|
67 self.builder._fetch_build = lambda build_number: None |
|
68 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
69 self.assertEqual(green_build, None) |
|
70 self.assertEqual(red_build, None) |
|
71 |
|
72 def test_flaky_tests(self): |
|
73 self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) |
|
74 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
75 self.assertEqual(green_build.revision(), 1009) |
|
76 self.assertEqual(red_build.revision(), 1010) |
|
77 |
|
78 def test_failure_and_flaky(self): |
|
79 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) |
|
80 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
81 self.assertEqual(green_build.revision(), 1003) |
|
82 self.assertEqual(red_build.revision(), 1004) |
|
83 |
|
84 def test_no_results(self): |
|
85 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) |
|
86 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
87 self.assertEqual(green_build.revision(), 1003) |
|
88 self.assertEqual(red_build.revision(), 1004) |
|
89 |
|
90 def test_failure_after_flaky(self): |
|
91 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) |
|
92 (green_build, red_build) = self.builder.find_failure_transition(self.builder.build(10)) |
|
93 self.assertEqual(green_build.revision(), 1006) |
|
94 self.assertEqual(red_build.revision(), 1007) |
|
95 |
|
96 def test_blameworthy_revisions(self): |
|
97 self.assertEqual(self.builder.blameworthy_revisions(10), [1004]) |
|
98 self.assertEqual(self.builder.blameworthy_revisions(10, look_back_limit=2), []) |
|
99 # Flakey test avoidance requires at least 2 red builds: |
|
100 self.assertEqual(self.builder.blameworthy_revisions(4), []) |
|
101 self.assertEqual(self.builder.blameworthy_revisions(4, avoid_flakey_tests=False), [1004]) |
|
102 # Green builder: |
|
103 self.assertEqual(self.builder.blameworthy_revisions(3), []) |
|
104 |
|
105 def test_build_caching(self): |
|
106 self.assertEqual(self.builder.build(10), self.builder.build(10)) |
|
107 |
|
108 def test_build_and_revision_for_filename(self): |
|
109 expectations = { |
|
110 "r47483 (1)/" : (47483, 1), |
|
111 "r47483 (1).zip" : (47483, 1), |
|
112 } |
|
113 for filename, revision_and_build in expectations.items(): |
|
114 self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) |
|
115 |
|
116 |
|
117 class LayoutTestResultsTest(unittest.TestCase): |
|
118 _example_results_html = """ |
|
119 <html> |
|
120 <head> |
|
121 <title>Layout Test Results</title> |
|
122 </head> |
|
123 <body> |
|
124 <p>Tests that had stderr output:</p> |
|
125 <table> |
|
126 <tr> |
|
127 <td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/accessibility/aria-activedescendant-crash.html">accessibility/aria-activedescendant-crash.html</a></td> |
|
128 <td><a href="accessibility/aria-activedescendant-crash-stderr.txt">stderr</a></td> |
|
129 </tr> |
|
130 <td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/http/tests/security/canvas-remote-read-svg-image.html">http/tests/security/canvas-remote-read-svg-image.html</a></td> |
|
131 <td><a href="http/tests/security/canvas-remote-read-svg-image-stderr.txt">stderr</a></td> |
|
132 </tr> |
|
133 </table><p>Tests that had no expected results (probably new):</p> |
|
134 <table> |
|
135 <tr> |
|
136 <td><a href="/var/lib/buildbot/build/gtk-linux-64-release/build/LayoutTests/fast/repaint/no-caret-repaint-in-non-content-editable-element.html">fast/repaint/no-caret-repaint-in-non-content-editable-element.html</a></td> |
|
137 <td><a href="fast/repaint/no-caret-repaint-in-non-content-editable-element-actual.txt">result</a></td> |
|
138 </tr> |
|
139 </table></body> |
|
140 </html> |
|
141 """ |
|
142 |
|
143 _expected_layout_test_results = { |
|
144 'Tests that had stderr output:' : [ |
|
145 'accessibility/aria-activedescendant-crash.html' |
|
146 ], |
|
147 'Tests that had no expected results (probably new):' : [ |
|
148 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' |
|
149 ] |
|
150 } |
|
151 def test_parse_layout_test_results(self): |
|
152 results = LayoutTestResults._parse_results_html(self._example_results_html) |
|
153 self.assertEqual(self._expected_layout_test_results, results) |
|
154 |
|
155 |
|
156 class BuildBotTest(unittest.TestCase): |
|
157 |
|
158 _example_one_box_status = ''' |
|
159 <table> |
|
160 <tr> |
|
161 <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> |
|
162 <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> |
|
163 <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> |
|
164 <tr> |
|
165 <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> |
|
166 <td class="LastBuild box" >no build</td> |
|
167 <td align="center" class="Activity building">building<br />< 1 min</td> |
|
168 <tr> |
|
169 <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> |
|
170 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> |
|
171 <td align="center" class="Activity idle">idle<br />3 pending</td> |
|
172 <tr> |
|
173 <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td> |
|
174 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td> |
|
175 <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td> |
|
176 </table> |
|
177 ''' |
|
178 _expected_example_one_box_parsings = [ |
|
179 { |
|
180 'is_green': True, |
|
181 'build_number' : 3693, |
|
182 'name': u'Windows Debug (Tests)', |
|
183 'built_revision': 47380, |
|
184 'activity': 'building', |
|
185 'pending_builds': 0, |
|
186 }, |
|
187 { |
|
188 'is_green': False, |
|
189 'build_number' : None, |
|
190 'name': u'SnowLeopard Intel Release', |
|
191 'built_revision': None, |
|
192 'activity': 'building', |
|
193 'pending_builds': 0, |
|
194 }, |
|
195 { |
|
196 'is_green': False, |
|
197 'build_number' : 654, |
|
198 'name': u'Qt Linux Release', |
|
199 'built_revision': 47383, |
|
200 'activity': 'idle', |
|
201 'pending_builds': 3, |
|
202 }, |
|
203 { |
|
204 'is_green': True, |
|
205 'build_number' : 2090, |
|
206 'name': u'Qt Windows 32-bit Debug', |
|
207 'built_revision': 60563, |
|
208 'activity': 'building', |
|
209 'pending_builds': 0, |
|
210 }, |
|
211 ] |
|
212 |
|
213 def test_status_parsing(self): |
|
214 buildbot = BuildBot() |
|
215 |
|
216 soup = BeautifulSoup(self._example_one_box_status) |
|
217 status_table = soup.find("table") |
|
218 input_rows = status_table.findAll('tr') |
|
219 |
|
220 for x in range(len(input_rows)): |
|
221 status_row = input_rows[x] |
|
222 expected_parsing = self._expected_example_one_box_parsings[x] |
|
223 |
|
224 builder = buildbot._parse_builder_status_from_row(status_row) |
|
225 |
|
226 # Make sure we aren't parsing more or less than we expect |
|
227 self.assertEquals(builder.keys(), expected_parsing.keys()) |
|
228 |
|
229 for key, expected_value in expected_parsing.items(): |
|
230 self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) |
|
231 |
|
232 def test_core_builder_methods(self): |
|
233 buildbot = BuildBot() |
|
234 |
|
235 # Override builder_statuses function to not touch the network. |
|
236 def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. |
|
237 return BuildBotTest._expected_example_one_box_parsings |
|
238 buildbot.builder_statuses = example_builder_statuses |
|
239 |
|
240 buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] |
|
241 self.assertEquals(buildbot.red_core_builders_names(), []) |
|
242 self.assertTrue(buildbot.core_builders_are_green()) |
|
243 |
|
244 buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] |
|
245 self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) |
|
246 self.assertFalse(buildbot.core_builders_are_green()) |
|
247 |
|
248 def test_builder_name_regexps(self): |
|
249 buildbot = BuildBot() |
|
250 |
|
251 # For complete testing, this list should match the list of builders at build.webkit.org: |
|
252 example_builders = [ |
|
253 {'name': u'Tiger Intel Release', }, |
|
254 {'name': u'Leopard Intel Release (Build)', }, |
|
255 {'name': u'Leopard Intel Release (Tests)', }, |
|
256 {'name': u'Leopard Intel Debug (Build)', }, |
|
257 {'name': u'Leopard Intel Debug (Tests)', }, |
|
258 {'name': u'SnowLeopard Intel Release (Build)', }, |
|
259 {'name': u'SnowLeopard Intel Release (Tests)', }, |
|
260 {'name': u'SnowLeopard Intel Leaks', }, |
|
261 {'name': u'Windows Release (Build)', }, |
|
262 {'name': u'Windows Release (Tests)', }, |
|
263 {'name': u'Windows Debug (Build)', }, |
|
264 {'name': u'Windows Debug (Tests)', }, |
|
265 {'name': u'GTK Linux 32-bit Release', }, |
|
266 {'name': u'GTK Linux 32-bit Debug', }, |
|
267 {'name': u'GTK Linux 64-bit Debug', }, |
|
268 {'name': u'GTK Linux 64-bit Release', }, |
|
269 {'name': u'Qt Linux Release', }, |
|
270 {'name': u'Qt Linux Release minimal', }, |
|
271 {'name': u'Qt Linux ARMv5 Release', }, |
|
272 {'name': u'Qt Linux ARMv7 Release', }, |
|
273 {'name': u'Qt Windows 32-bit Release', }, |
|
274 {'name': u'Qt Windows 32-bit Debug', }, |
|
275 {'name': u'Chromium Linux Release', }, |
|
276 {'name': u'Chromium Mac Release', }, |
|
277 {'name': u'Chromium Win Release', }, |
|
278 {'name': u'Chromium Linux Release (Tests)', }, |
|
279 {'name': u'Chromium Mac Release (Tests)', }, |
|
280 {'name': u'Chromium Win Release (Tests)', }, |
|
281 {'name': u'New run-webkit-tests', }, |
|
282 ] |
|
283 name_regexps = [ |
|
284 "SnowLeopard.*Build", |
|
285 "SnowLeopard.*Test", |
|
286 "Leopard", |
|
287 "Tiger", |
|
288 "Windows.*Build", |
|
289 "GTK.*32", |
|
290 "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. |
|
291 "Qt", |
|
292 "Chromium.*Release$", |
|
293 ] |
|
294 expected_builders = [ |
|
295 {'name': u'Tiger Intel Release', }, |
|
296 {'name': u'Leopard Intel Release (Build)', }, |
|
297 {'name': u'Leopard Intel Release (Tests)', }, |
|
298 {'name': u'Leopard Intel Debug (Build)', }, |
|
299 {'name': u'Leopard Intel Debug (Tests)', }, |
|
300 {'name': u'SnowLeopard Intel Release (Build)', }, |
|
301 {'name': u'SnowLeopard Intel Release (Tests)', }, |
|
302 {'name': u'Windows Release (Build)', }, |
|
303 {'name': u'Windows Debug (Build)', }, |
|
304 {'name': u'GTK Linux 32-bit Release', }, |
|
305 {'name': u'GTK Linux 32-bit Debug', }, |
|
306 {'name': u'GTK Linux 64-bit Debug', }, |
|
307 {'name': u'Qt Linux Release', }, |
|
308 {'name': u'Qt Linux Release minimal', }, |
|
309 {'name': u'Qt Linux ARMv5 Release', }, |
|
310 {'name': u'Qt Linux ARMv7 Release', }, |
|
311 {'name': u'Qt Windows 32-bit Release', }, |
|
312 {'name': u'Qt Windows 32-bit Debug', }, |
|
313 {'name': u'Chromium Linux Release', }, |
|
314 {'name': u'Chromium Mac Release', }, |
|
315 {'name': u'Chromium Win Release', }, |
|
316 ] |
|
317 |
|
318 # This test should probably be updated if the default regexp list changes |
|
319 self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) |
|
320 |
|
321 builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) |
|
322 self.assertEquals(builders, expected_builders) |
|
323 |
|
324 def test_builder_with_name(self): |
|
325 buildbot = BuildBot() |
|
326 |
|
327 builder = buildbot.builder_with_name("Test Builder") |
|
328 self.assertEqual(builder.name(), "Test Builder") |
|
329 self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder") |
|
330 self.assertEqual(builder.url_encoded_name(), "Test%20Builder") |
|
331 self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder") |
|
332 |
|
333 # Override _fetch_xmlrpc_build_dictionary function to not touch the network. |
|
334 def mock_fetch_xmlrpc_build_dictionary(self, build_number): |
|
335 build_dictionary = { |
|
336 "revision" : 2 * build_number, |
|
337 "number" : int(build_number), |
|
338 "results" : build_number % 2, # 0 means pass |
|
339 } |
|
340 return build_dictionary |
|
341 buildbot._fetch_xmlrpc_build_dictionary = mock_fetch_xmlrpc_build_dictionary |
|
342 |
|
343 build = builder.build(10) |
|
344 self.assertEqual(build.builder(), builder) |
|
345 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10") |
|
346 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29") |
|
347 self.assertEqual(build.revision(), 20) |
|
348 self.assertEqual(build.is_green(), True) |
|
349 |
|
350 build = build.previous_build() |
|
351 self.assertEqual(build.builder(), builder) |
|
352 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9") |
|
353 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29") |
|
354 self.assertEqual(build.revision(), 18) |
|
355 self.assertEqual(build.is_green(), False) |
|
356 |
|
357 self.assertEqual(builder.build(None), None) |
|
358 |
|
359 _example_directory_listing = ''' |
|
360 <h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1> |
|
361 |
|
362 <table> |
|
363 <thead> |
|
364 <tr> |
|
365 <th>Filename</th> |
|
366 <th>Size</th> |
|
367 <th>Content type</th> |
|
368 <th>Content encoding</th> |
|
369 </tr> |
|
370 </thead> |
|
371 <tbody> |
|
372 <tr class="odd"> |
|
373 <td><a href="r47483%20%281%29/">r47483 (1)/</a></td> |
|
374 <td></td> |
|
375 <td>[Directory]</td> |
|
376 <td></td> |
|
377 </tr> |
|
378 <tr class="odd"> |
|
379 <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td> |
|
380 <td>89K</td> |
|
381 <td>[application/zip]</td> |
|
382 <td></td> |
|
383 </tr> |
|
384 ''' |
|
385 _expected_files = [ |
|
386 { |
|
387 "filename" : "r47483 (1)/", |
|
388 "size" : "", |
|
389 "type" : "[Directory]", |
|
390 "encoding" : "", |
|
391 }, |
|
392 { |
|
393 "filename" : "r47484 (2).zip", |
|
394 "size" : "89K", |
|
395 "type" : "[application/zip]", |
|
396 "encoding" : "", |
|
397 }, |
|
398 ] |
|
399 |
|
400 def test_parse_build_to_revision_map(self): |
|
401 buildbot = BuildBot() |
|
402 files = buildbot._parse_twisted_directory_listing(self._example_directory_listing) |
|
403 self.assertEqual(self._expected_files, files) |
|
404 |
|
405 # Revision, is_green |
|
406 # Ordered from newest (highest number) to oldest. |
|
407 fake_builder1 = [ |
|
408 [2, False], |
|
409 [1, True], |
|
410 ] |
|
411 fake_builder2 = [ |
|
412 [2, False], |
|
413 [1, True], |
|
414 ] |
|
415 fake_builders = [ |
|
416 fake_builder1, |
|
417 fake_builder2, |
|
418 ] |
|
419 def _build_from_fake(self, fake_builder, index): |
|
420 if index >= len(fake_builder): |
|
421 return None |
|
422 fake_build = fake_builder[index] |
|
423 build = Build( |
|
424 builder=fake_builder, |
|
425 build_number=index, |
|
426 revision=fake_build[0], |
|
427 is_green=fake_build[1], |
|
428 ) |
|
429 def mock_previous_build(): |
|
430 return self._build_from_fake(fake_builder, index + 1) |
|
431 build.previous_build = mock_previous_build |
|
432 return build |
|
433 |
|
434 def _fake_builds_at_index(self, index): |
|
435 return [self._build_from_fake(builder, index) for builder in self.fake_builders] |
|
436 |
|
437 def test_last_green_revision(self): |
|
438 buildbot = BuildBot() |
|
439 def mock_builds_from_builders(only_core_builders): |
|
440 return self._fake_builds_at_index(0) |
|
441 buildbot._latest_builds_from_builders = mock_builds_from_builders |
|
442 self.assertEqual(buildbot.last_green_revision(), 1) |
|
443 |
|
444 |
|
445 if __name__ == '__main__': |
|
446 unittest.main() |