| changeset 0 | 4f2f89ce4247 |
| -1:000000000000 | 0:4f2f89ce4247 |
|---|---|
1 /* |
|
2 * Copyright (C) 2007 Apple Inc. All rights reserved. |
|
3 * |
|
4 * Redistribution and use in source and binary forms, with or without |
|
5 * modification, are permitted provided that the following conditions |
|
6 * are met: |
|
7 * 1. Redistributions of source code must retain the above copyright |
|
8 * notice, this list of conditions and the following disclaimer. |
|
9 * 2. Redistributions in binary form must reproduce the above copyright |
|
10 * notice, this list of conditions and the following disclaimer in the |
|
11 * documentation and/or other materials provided with the distribution. |
|
12 * |
|
13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY |
|
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
|
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR |
|
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
|
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
|
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
|
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY |
|
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
24 */ |
|
25 |
|
26 function sunspiderCompareResults(output1, output2) |
|
27 { |
|
28 var count1 = output1.length; |
|
29 var count2 = output2.length; |
|
30 |
|
31 var itemTotals1 = {}; |
|
32 itemTotals1.length = count1; |
|
33 |
|
34 var total1 = 0; |
|
35 var categoryTotals1 = {}; |
|
36 var testTotalsByCategory1 = {}; |
|
37 |
|
38 var mean1 = 0; |
|
39 var categoryMeans1 = {}; |
|
40 var testMeansByCategory1 = {}; |
|
41 |
|
42 var stdDev1 = 0; |
|
43 var categoryStdDevs1 = {}; |
|
44 var testStdDevsByCategory1 = {}; |
|
45 |
|
46 var stdErr1 = 0; |
|
47 var categoryStdErrs1 = {}; |
|
48 var testStdErrsByCategory1 = {}; |
|
49 |
|
50 var itemTotals2 = {}; |
|
51 itemTotals2.length = count2; |
|
52 |
|
53 var total2 = 0; |
|
54 var categoryTotals2 = {}; |
|
55 var testTotalsByCategory2 = {}; |
|
56 |
|
57 var mean2 = 0; |
|
58 var categoryMeans2 = {}; |
|
59 var testMeansByCategory2 = {}; |
|
60 |
|
61 var stdDev2 = 0; |
|
62 var categoryStdDevs2 = {}; |
|
63 var testStdDevsByCategory2 = {}; |
|
64 |
|
65 var stdErr2 = 0; |
|
66 var categoryStdErrs2 = {}; |
|
67 var testStdErrsByCategory2 = {}; |
|
68 |
|
69 function initialize() |
|
70 { |
|
71 itemTotals1 = {total: []}; |
|
72 |
|
73 for (var i = 0; i < categories.length; i++) { |
|
74 var category = categories[i]; |
|
75 itemTotals1[category] = []; |
|
76 categoryTotals1[category] = 0; |
|
77 testTotalsByCategory1[category] = {}; |
|
78 categoryMeans1[category] = 0; |
|
79 testMeansByCategory1[category] = {}; |
|
80 categoryStdDevs1[category] = 0; |
|
81 testStdDevsByCategory1[category] = {}; |
|
82 categoryStdErrs1[category] = 0; |
|
83 testStdErrsByCategory1[category] = {}; |
|
84 } |
|
85 |
|
86 for (var i = 0; i < tests.length; i++) { |
|
87 var test = tests[i]; |
|
88 itemTotals1[test] = []; |
|
89 var category = test.replace(/-.*/, ""); |
|
90 testTotalsByCategory1[category][test] = 0; |
|
91 testMeansByCategory1[category][test] = 0; |
|
92 testStdDevsByCategory1[category][test] = 0; |
|
93 testStdErrsByCategory1[category][test] = 0; |
|
94 } |
|
95 |
|
96 for (var i = 0; i < count1; i++) { |
|
97 itemTotals1["total"][i] = 0; |
|
98 for (var category in categoryTotals1) { |
|
99 itemTotals1[category][i] = 0; |
|
100 for (var test in testTotalsByCategory1[category]) { |
|
101 itemTotals1[test][i] = 0; |
|
102 } |
|
103 } |
|
104 } |
|
105 |
|
106 itemTotals2 = {total: []}; |
|
107 |
|
108 for (var i = 0; i < categories.length; i++) { |
|
109 var category = categories[i]; |
|
110 itemTotals2[category] = []; |
|
111 categoryTotals2[category] = 0; |
|
112 testTotalsByCategory2[category] = {}; |
|
113 categoryMeans2[category] = 0; |
|
114 testMeansByCategory2[category] = {}; |
|
115 categoryStdDevs2[category] = 0; |
|
116 testStdDevsByCategory2[category] = {}; |
|
117 categoryStdErrs2[category] = 0; |
|
118 testStdErrsByCategory2[category] = {}; |
|
119 } |
|
120 |
|
121 for (var i = 0; i < tests.length; i++) { |
|
122 var test = tests[i]; |
|
123 itemTotals2[test] = []; |
|
124 var category = test.replace(/-.*/, ""); |
|
125 testTotalsByCategory2[category][test] = 0; |
|
126 testMeansByCategory2[category][test] = 0; |
|
127 testStdDevsByCategory2[category][test] = 0; |
|
128 testStdErrsByCategory2[category][test] = 0; |
|
129 } |
|
130 |
|
131 for (var i = 0; i < count2; i++) { |
|
132 itemTotals2["total"][i] = 0; |
|
133 for (var category in categoryTotals2) { |
|
134 itemTotals2[category][i] = 0; |
|
135 for (var test in testTotalsByCategory2[category]) { |
|
136 itemTotals2[test][i] = 0; |
|
137 } |
|
138 } |
|
139 } |
|
140 |
|
141 } |
|
142 |
|
143 function computeItemTotals(output, itemTotals) |
|
144 { |
|
145 for (var i = 0; i < output.length; i++) { |
|
146 var result = output[i]; |
|
147 for (var test in result) { |
|
148 var time = result[test]; |
|
149 var category = test.replace(/-.*/, ""); |
|
150 itemTotals["total"][i] += time; |
|
151 itemTotals[category][i] += time; |
|
152 itemTotals[test][i] += time; |
|
153 } |
|
154 } |
|
155 } |
|
156 |
|
157 function computeTotals(output, categoryTotals, testTotalsByCategory) |
|
158 { |
|
159 var total = 0; |
|
160 |
|
161 for (var i = 0; i < output.length; i++) { |
|
162 var result = output[i]; |
|
163 for (var test in result) { |
|
164 var time = result[test]; |
|
165 var category = test.replace(/-.*/, ""); |
|
166 total += time; |
|
167 categoryTotals[category] += time; |
|
168 testTotalsByCategory[category][test] += time; |
|
169 } |
|
170 } |
|
171 |
|
172 return total; |
|
173 } |
|
174 |
|
175 function computeMeans(count, total, categoryTotals, categoryMeans, testTotalsByCategory, testMeansByCategory) |
|
176 { |
|
177 var mean = total / count; |
|
178 for (var category in categoryTotals) { |
|
179 categoryMeans[category] = categoryTotals[category] / count; |
|
180 for (var test in testTotalsByCategory[category]) { |
|
181 testMeansByCategory[category][test] = testTotalsByCategory[category][test] / count; |
|
182 } |
|
183 } |
|
184 return mean; |
|
185 } |
|
186 |
|
187 function standardDeviation(mean, items) |
|
188 { |
|
189 var deltaSquaredSum = 0; |
|
190 for (var i = 0; i < items.length; i++) { |
|
191 var delta = items[i] - mean; |
|
192 deltaSquaredSum += delta * delta; |
|
193 } |
|
194 variance = deltaSquaredSum / (items.length - 1); |
|
195 return Math.sqrt(variance); |
|
196 } |
|
197 |
|
198 function computeStdDevs(mean, itemTotals, categoryStdDevs, categoryMeans, testStdDevsByCategory, testMeansByCategory) |
|
199 { |
|
200 var stdDev = standardDeviation(mean, itemTotals["total"]); |
|
201 for (var category in categoryStdDevs) { |
|
202 categoryStdDevs[category] = standardDeviation(categoryMeans[category], itemTotals[category]); |
|
203 } |
|
204 for (var category in categoryStdDevs) { |
|
205 for (var test in testStdDevsByCategory[category]) { |
|
206 testStdDevsByCategory[category][test] = standardDeviation(testMeansByCategory[category][test], itemTotals[test]); |
|
207 } |
|
208 } |
|
209 return stdDev; |
|
210 } |
|
211 |
|
212 function computeStdErrors(count, stdDev, categoryStdErrs, categoryStdDevs, testStdErrsByCategory, testStdDevsByCategory) |
|
213 { |
|
214 var sqrtCount = Math.sqrt(count); |
|
215 |
|
216 var stdErr = stdDev / sqrtCount; |
|
217 for (var category in categoryStdErrs) { |
|
218 categoryStdErrs[category] = categoryStdDevs[category] / sqrtCount; |
|
219 } |
|
220 for (var category in categoryStdDevs) { |
|
221 for (var test in testStdErrsByCategory[category]) { |
|
222 testStdErrsByCategory[category][test] = testStdDevsByCategory[category][test] / sqrtCount; |
|
223 } |
|
224 } |
|
225 |
|
226 return stdErr; |
|
227 } |
|
228 |
|
229 var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09, 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04, 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96]; |
|
230 var tMax = tDistribution.length; |
|
231 var tLimit = 1.96; |
|
232 |
|
233 function tDist(n) |
|
234 { |
|
235 if (n > tMax) |
|
236 return tLimit; |
|
237 return tDistribution[n]; |
|
238 } |
|
239 |
|
240 |
|
241 function formatMean(meanWidth, mean, stdErr, count) |
|
242 { |
|
243 var meanString = mean.toFixed(1).toString(); |
|
244 while (meanString.length < meanWidth) { |
|
245 meanString = " " + meanString; |
|
246 } |
|
247 |
|
248 var error = "+/- " + ((tDist(count) * stdErr / mean) * 100).toFixed(1) + "% "; |
|
249 |
|
250 return meanString + "ms " + error; |
|
251 } |
|
252 |
|
253 function computeLabelWidth() |
|
254 { |
|
255 var width = "Total".length; |
|
256 for (var category in categoryMeans1) { |
|
257 if (category.length + 2 > width) |
|
258 width = category.length + 2; |
|
259 } |
|
260 for (var i = 0; i < tests.length; i++) { |
|
261 var shortName = tests[i].replace(/^[^-]*-/, ""); |
|
262 if (shortName.length + 4 > width) |
|
263 width = shortName.length + 4; |
|
264 } |
|
265 |
|
266 return width; |
|
267 } |
|
268 |
|
269 function computeMeanWidth(mean, categoryMeans, testMeansByCategory) |
|
270 { |
|
271 var width = mean.toFixed(1).toString().length; |
|
272 for (var category in categoryMeans) { |
|
273 var candidate = categoryMeans[category].toFixed(1).toString().length; |
|
274 if (candidate > width) |
|
275 width = candidate; |
|
276 for (var test in testMeansByCategory[category]) { |
|
277 var candidate = testMeansByCategory[category][test].toFixed(1).toString().length; |
|
278 if (candidate > width) |
|
279 width = candidate; |
|
280 } |
|
281 } |
|
282 |
|
283 return width; |
|
284 } |
|
285 |
|
286 function pad(str, n) |
|
287 { |
|
288 while (str.length < n) { |
|
289 str += " "; |
|
290 } |
|
291 return str; |
|
292 } |
|
293 |
|
294 function resultLine(labelWidth, indent, label, meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2) |
|
295 { |
|
296 result = pad("", indent); |
|
297 result += label + ": "; |
|
298 result = pad(result, labelWidth + 2); |
|
299 |
|
300 var t = (mean1 - mean2) / (Math.sqrt((stdErr1 * stdErr1) + (stdErr2 * stdErr2))); |
|
301 var df = count1 + count2 - 2; |
|
302 |
|
303 var statisticallySignificant = (Math.abs(t) > tDist(df+1)); |
|
304 var diff = mean2 - mean1; |
|
305 var percentage = 100 * diff / mean1; |
|
306 var isFaster = diff < 0; |
|
307 var probablySame = (percentage < 0.1) && !statisticallySignificant; |
|
308 var ratio = isFaster ? (mean1 / mean2) : (mean2 / mean1); |
|
309 var fixedRatio = (ratio < 1.2) ? ratio.toFixed(3).toString() : ((ratio < 10) ? ratio.toFixed(2).toString() : ratio.toFixed(1).toString()); |
|
310 var formattedRatio = isFaster ? fixedRatio + "x as fast" : "*" + fixedRatio + "x as slow*"; |
|
311 |
|
312 var diffSummary; |
|
313 var diffDetail; |
|
314 |
|
315 if (probablySame) { |
|
316 diffSummary = "-"; |
|
317 diffDetail = ""; |
|
318 } else if (!statisticallySignificant) { |
|
319 diffSummary = "??"; |
|
320 diffDetail = " not conclusive: might be " + formattedRatio; |
|
321 } else { |
|
322 diffSummary = formattedRatio; |
|
323 diffDetail = " significant"; |
|
324 } |
|
325 |
|
326 return result + pad(diffSummary, 18) + formatMean(meanWidth1, mean1, stdErr1, count1) + " " + formatMean(meanWidth2, mean2, stdErr2, count2) + diffDetail; |
|
327 } |
|
328 |
|
329 function printOutput() |
|
330 { |
|
331 var labelWidth = computeLabelWidth(); |
|
332 var meanWidth1 = computeMeanWidth(mean1, categoryMeans1, testMeansByCategory1); |
|
333 var meanWidth2 = computeMeanWidth(mean2, categoryMeans2, testMeansByCategory2); |
|
334 |
|
335 print("\n"); |
|
336 var header = "TEST"; |
|
337 while (header.length < labelWidth) |
|
338 header += " "; |
|
339 header += " COMPARISON FROM TO DETAILS"; |
|
340 print(header); |
|
341 print(""); |
|
342 print("============================================================================="); |
|
343 print(""); |
|
344 print(resultLine(labelWidth, 0, "** TOTAL **", meanWidth1, mean1, stdErr1, meanWidth2, mean2, stdErr2)); |
|
345 print(""); |
|
346 print("============================================================================="); |
|
347 |
|
348 for (var category in categoryMeans1) { |
|
349 print(""); |
|
350 print(resultLine(labelWidth, 2, category, |
|
351 meanWidth1, categoryMeans1[category], categoryStdErrs1[category], |
|
352 meanWidth2, categoryMeans2[category], categoryStdErrs2[category])); |
|
353 for (var test in testMeansByCategory1[category]) { |
|
354 var shortName = test.replace(/^[^-]*-/, ""); |
|
355 print(resultLine(labelWidth, 4, shortName, |
|
356 meanWidth1, testMeansByCategory1[category][test], testStdErrsByCategory1[category][test], |
|
357 meanWidth2, testMeansByCategory2[category][test], testStdErrsByCategory2[category][test])); |
|
358 } |
|
359 } |
|
360 } |
|
361 |
|
362 initialize(); |
|
363 |
|
364 computeItemTotals(output1, itemTotals1); |
|
365 computeItemTotals(output2, itemTotals2); |
|
366 |
|
367 total1 = computeTotals(output1, categoryTotals1, testTotalsByCategory1); |
|
368 total2 = computeTotals(output2, categoryTotals2, testTotalsByCategory2); |
|
369 |
|
370 mean1 = computeMeans(count1, total1, categoryTotals1, categoryMeans1, testTotalsByCategory1, testMeansByCategory1); |
|
371 mean2 = computeMeans(count2, total2, categoryTotals2, categoryMeans2, testTotalsByCategory2, testMeansByCategory2); |
|
372 |
|
373 stdDev1 = computeStdDevs(mean1, itemTotals1, categoryStdDevs1, categoryMeans1, testStdDevsByCategory1, testMeansByCategory1); |
|
374 stdDev2 = computeStdDevs(mean2, itemTotals2, categoryStdDevs2, categoryMeans2, testStdDevsByCategory2, testMeansByCategory2); |
|
375 |
|
376 stdErr1 = computeStdErrors(count1, stdDev1, categoryStdErrs1, categoryStdDevs1, testStdErrsByCategory1, testStdDevsByCategory1); |
|
377 stdErr2 = computeStdErrors(count2, stdDev2, categoryStdErrs2, categoryStdDevs2, testStdErrsByCategory2, testStdDevsByCategory2); |
|
378 |
|
379 printOutput(); |
|
380 } |