Diff
Modified: trunk/LayoutTests/ChangeLog (181578 => 181579)
--- trunk/LayoutTests/ChangeLog 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/LayoutTests/ChangeLog 2015-03-16 21:52:40 UTC (rev 181579)
@@ -1,3 +1,14 @@
+2015-03-16 Alexey Proskuryakov <a...@apple.com>
+
+ Add tolerance to WebAudio tests
+ https://bugs.webkit.org/show_bug.cgi?id=142676
+
+ Reviewed by Tim Horton.
+
+ * fast/harness/results.html: Display a diff link for audio tests, as we now have the diff.
+
+ * platform/mac/TestExpectations: Unmark tests that should now pass everywhere.
+
2015-03-16 Chris Dumez <cdu...@apple.com>
Make DatabaseContext suspendable if there is no pending database activity
Modified: trunk/LayoutTests/fast/harness/results.html (181578 => 181579)
--- trunk/LayoutTests/fast/harness/results.html 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/LayoutTests/fast/harness/results.html 2015-03-16 21:52:40 UTC (rev 181579)
@@ -659,6 +659,7 @@
if (actual.indexOf('AUDIO') != -1) {
row += resultLink(testPrefix, '-expected.wav', 'expected audio');
row += resultLink(testPrefix, '-actual.wav', 'actual audio');
+ row += resultLink(testPrefix, '-diff.txt', 'textual diff');
}
if (actual.indexOf('MISSING') != -1) {
Modified: trunk/LayoutTests/platform/mac/TestExpectations (181578 => 181579)
--- trunk/LayoutTests/platform/mac/TestExpectations 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/LayoutTests/platform/mac/TestExpectations 2015-03-16 21:52:40 UTC (rev 181579)
@@ -617,7 +617,6 @@
webaudio/mediastreamaudiosourcenode.html [ Failure ]
webaudio/codec-tests/vorbis/ [ WontFix ]
webkit.org/b/119467 webaudio/audiobuffersource-loop-points.html [ Skip ]
-[ Mavericks ] webaudio/codec-tests/wav/24bit-22khz-resample.html [ Failure ]
# Text Autosizing is not enabled.
webkit.org/b/84186 fast/text-autosizing
@@ -759,14 +758,10 @@
# webkit.org/b/100846, webkit.org/b/136715
inspector-protocol/debugger
-webkit.org/b/123490 [ Mavericks ] webaudio/oscillator-sawtooth.html [ Pass Failure ]
-
webkit.org/b/124311 compositing/regions/transform-transparent-positioned-video-inside-region.html [ ImageOnlyFailure ]
webkit.org/b/124321 [ Mavericks ] animations/resume-after-page-cache.html [ Pass Failure ]
-webkit.org/b/121646 webaudio/delaynode-max-default-delay.html [ Pass Failure ]
-
# These fast/forms/select tests open a pop-up menu (visible on screen even when using run-webkit-tests), and get stuck in its nested event loop.
webkit.org/b/87748 fast/forms/select/optgroup-clicking.html [ Skip ]
webkit.org/b/73304 fast/forms/select/menulist-popup-crash.html [ Skip ]
@@ -1090,10 +1085,6 @@
webkit.org/b/137157 [ Release ] inspector/page/main-frame-resource.html [ Pass Failure ]
[ Debug ] inspector/page/main-frame-resource.html [ Pass Failure Slow ]
-# FIXME: Needs bugzilla (<rdar://problem/15393179>)
-[ Yosemite+ ] webaudio/codec-tests/wav/24bit-22khz-resample.html [ Failure ]
-[ Yosemite+ ] webaudio/oscillator-sawtooth.html [ Pass Failure ]
-
webkit.org/b/137505 media/track/track-forced-subtitles-in-band.html [ Failure Pass ]
# FIXME: Needs bugzilla (<rdar://problem/15971968>)
Modified: trunk/Tools/ChangeLog (181578 => 181579)
--- trunk/Tools/ChangeLog 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/Tools/ChangeLog 2015-03-16 21:52:40 UTC (rev 181579)
@@ -1,5 +1,26 @@
2015-03-16 Alexey Proskuryakov <a...@apple.com>
+ Add tolerance to WebAudio tests
+ https://bugs.webkit.org/show_bug.cgi?id=142676
+
+ Reviewed by Tim Horton.
+
+ * Scripts/webkitpy/common/wavediff.py: Added. Based on Jer Noble's work.
+
+ * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
+ (TestResultWriter.create_audio_diff_and_write_result):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (FailureAudio.write_failure):
+ * Scripts/webkitpy/port/base.py:
+ (Port.do_audio_results_differ):
+ Diff audio failures.
+
+ * Scripts/webkitpy/port/test.py: Added a test for the tolerance, fixed existing
+ tests to use real parseable WAV data, and got rid of base64, which there didn't
+ seem to have been any reason for.
+
+2015-03-16 Alexey Proskuryakov <a...@apple.com>
+
[Mac] fast/forms/text-control-intrinsic-widths.html fails when MS Office is installed
https://bugs.webkit.org/show_bug.cgi?id=142720
Added: trunk/Tools/Scripts/webkitpy/common/wavediff.py (0 => 181579)
--- trunk/Tools/Scripts/webkitpy/common/wavediff.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/common/wavediff.py 2015-03-16 21:52:40 UTC (rev 181579)
@@ -0,0 +1,112 @@
+# Copyright (C) 2015 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import StringIO
+import struct
+import sys
+import tempfile
+import wave
+
+
+class WaveDiff(object):
+ _paramNames = ('Number of channels', 'Sample width', 'Sample rate', 'Number of frames', 'Compression type', 'Compression name')
+
+ # Audio effect processing is intrinsically imprecise, so we need to always allow tolerance.
+ _tolerance = 1
+
+ def __init__(self, in1, in2):
+ if isinstance(in1, file):
+ waveFile1 = wave.open(in1, 'rb')
+ else:
+ waveFile1 = wave.open(StringIO.StringIO(in1), 'rb')
+ if isinstance(in2, file):
+ waveFile1 = wave.open(in2, 'rb')
+ else:
+ waveFile2 = wave.open(StringIO.StringIO(in2), 'rb')
+
+ params1 = waveFile1.getparams()
+ params2 = waveFile2.getparams()
+
+ self._diff = ''
+
+ self._filesAreIdentical = not sum(map(self._diffParam, params1, params2, self._paramNames))
+ self._filesAreIdenticalWithinTolerance = self._filesAreIdentical
+
+ if not self._filesAreIdentical:
+ return
+
+ # Metadata is identical, compare the content now.
+
+ channelCount1 = waveFile1.getnchannels()
+ frameCount1 = waveFile1.getnframes()
+ sampleWidth1 = waveFile1.getsampwidth()
+ channelCount2 = waveFile2.getnchannels()
+ frameCount2 = waveFile2.getnframes()
+ sampleWidth2 = waveFile2.getsampwidth()
+
+ allData1 = self._readSamples(waveFile1, sampleWidth1, frameCount1 * channelCount1)
+ allData2 = self._readSamples(waveFile2, sampleWidth2, frameCount2 * channelCount2)
+ results = map(self._diffSample, allData1, allData2, xrange(max(frameCount1 * channelCount1, frameCount2 * channelCount2)))
+
+ cumulativeSampleDiff = sum(results)
+ differingSampleCount = len(filter(bool, results))
+ self._filesAreIdentical = not differingSampleCount
+ self._filesAreIdenticalWithinTolerance = not len(filter(lambda x: x > self._tolerance, results))
+
+ if differingSampleCount:
+ self._diff += '\n'
+ self._diff += 'Total differing samples: %d\n' % differingSampleCount
+ self._diff += 'Percentage of differing samples: %0.3f%%\n' % (100 * float(differingSampleCount) / max(frameCount1, frameCount2))
+ self._diff += 'Cumulative sample difference: %d\n' % cumulativeSampleDiff
+ self._diff += 'Average sample difference: %f\n' % (float(cumulativeSampleDiff) / differingSampleCount)
+
+ def _diffParam(self, param1, param2, paramName):
+ if param1 == param2:
+ return False
+ self._diff += paramName + '\n'
+ self._diff += '< %s\n' % str(param1)
+ self._diff += '---\n'
+ self._diff += '> %s\n' % str(param2)
+ return True
+
+ @staticmethod
+ def _readSamples(file, sampleWidth, nSamples):
+ allFrames = file.readframes(nSamples)
+ unpackFormat = 'b' if sampleWidth == 1 else 'h'
+ return struct.unpack('<%d%s' % (nSamples, unpackFormat), allFrames)
+
+ def _diffSample(self, data1, data2, i):
+ if (data1 != data2):
+ self._diff += 'Sample #%d\n' % i
+ self._diff += '< %d\n' % data1
+ self._diff += '---\n'
+ self._diff += '> %d\n' % data2
+ return abs(data1 - data2)
+
+ def filesAreIdentical(self):
+ return self._filesAreIdentical
+
+ def filesAreIdenticalWithinTolerance(self):
+ return self._filesAreIdenticalWithinTolerance
+
+ def diffText(self):
+ return self._diff
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py (181578 => 181579)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2015-03-16 21:52:40 UTC (rev 181579)
@@ -29,6 +29,7 @@
import logging
+from webkitpy.common.wavediff import WaveDiff
from webkitpy.layout_tests.models import test_failures
@@ -161,6 +162,10 @@
def write_audio_files(self, actual_audio, expected_audio):
self.write_output_files('.wav', actual_audio, expected_audio)
+ def create_audio_diff_and_write_result(self, actual_audio, expected_audio):
+ diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + '.txt')
+ self._write_text_file(diff_filename, WaveDiff(expected_audio, actual_audio).diffText())
+
def write_image_files(self, actual_image, expected_image):
self.write_output_files('.png', actual_image, expected_image)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (181578 => 181579)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2015-03-16 21:52:40 UTC (rev 181579)
@@ -122,6 +122,7 @@
class FailureAudio(TestFailure):
def write_failure(self, writer, driver_output, expected_driver_output, port):
writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
+ writer.create_audio_diff_and_write_result(driver_output.audio, expected_driver_output.audio)
class FailureTimeout(TestFailure):
Modified: trunk/Tools/Scripts/webkitpy/port/base.py (181578 => 181579)
--- trunk/Tools/Scripts/webkitpy/port/base.py 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/Tools/Scripts/webkitpy/port/base.py 2015-03-16 21:52:40 UTC (rev 181579)
@@ -50,6 +50,7 @@
from webkitpy.common.system import path
from webkitpy.common.system.executive import ScriptError
from webkitpy.common.system.systemhost import SystemHost
+from webkitpy.common.wavediff import WaveDiff
from webkitpy.common.webkit_finder import WebKitFinder
from webkitpy.layout_tests.models.test_configuration import TestConfiguration
from webkitpy.port import config as port_config
@@ -282,7 +283,9 @@
return expected_text != actual_text
def do_audio_results_differ(self, expected_audio, actual_audio):
- return expected_audio != actual_audio
+ if expected_audio == actual_audio:
+ return False
+ return not WaveDiff(expected_audio, actual_audio).filesAreIdenticalWithinTolerance()
def diff_image(self, expected_contents, actual_contents, tolerance=None):
"""Compare two images and return a tuple of an image diff, a percentage difference (0-100), and an error string.
Modified: trunk/Tools/Scripts/webkitpy/port/test.py (181578 => 181579)
--- trunk/Tools/Scripts/webkitpy/port/test.py 2015-03-16 21:24:48 UTC (rev 181578)
+++ trunk/Tools/Scripts/webkitpy/port/test.py 2015-03-16 21:52:40 UTC (rev 181579)
@@ -26,7 +26,6 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import base64
import sys
import time
@@ -99,7 +98,7 @@
#
# These numbers may need to be updated whenever we add or delete tests.
#
-TOTAL_TESTS = 71
+TOTAL_TESTS = 72
TOTAL_SKIPS = 9
TOTAL_RETRIES = 14
@@ -107,6 +106,9 @@
UNEXPECTED_FAILURES = 17
def unit_test_list():
+ silent_audio = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+ silent_audio_with_single_bit_difference = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+ audio2 = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
tests = TestList()
tests.add('failures/expected/crash.html', crash=True)
tests.add('failures/expected/exception.html', exception=True)
@@ -120,7 +122,7 @@
actual_checksum='image_checksum_fail-checksum',
actual_image='image_checksum_fail-png')
tests.add('failures/expected/audio.html',
- actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
+ actual_audio=silent_audio, expected_audio=audio2,
actual_text=None, expected_text=None,
actual_image=None, expected_image=None,
actual_checksum=None)
@@ -181,10 +183,15 @@
tests.add('passes/error.html', error='stuff going to stderr')
tests.add('passes/image.html')
tests.add('passes/audio.html',
- actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
+ actual_audio=silent_audio, expected_audio=silent_audio,
actual_text=None, expected_text=None,
actual_image=None, expected_image=None,
actual_checksum=None)
+ tests.add('passes/audio-tolerance.html',
+ actual_audio=silent_audio_with_single_bit_difference, expected_audio=silent_audio,
+ actual_text=None, expected_text=None,
+ actual_image=None, expected_image=None,
+ actual_checksum=None)
tests.add('passes/platform_image.html')
tests.add('passes/checksum_in_image.html',
expected_image='tEXtchecksum\x00checksum_in_image-checksum')
@@ -558,7 +565,7 @@
actual_text = actual_text + ' ' + ' '.join(test_args)
if test.actual_audio:
- audio = base64.b64decode(test.actual_audio)
+ audio = test.actual_audio
crashed_process_name = None
crashed_pid = None
if test.crash:
