| # Copyright (C) 2010 Google Inc. All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| """A helper class for reading in and dealing with tests expectations |
| for layout tests. |
| """ |
| |
| import logging |
| import re |
| |
| from webkitpy.layout_tests.models.test_configuration import TestConfigurationConverter |
| |
| _log = logging.getLogger(__name__) |
| |
| |
| # Test expectation and specifier constants. |
| # |
| # FIXME: range() starts with 0 which makes if expectation checks harder |
| # as PASS is 0. |
| (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WONTFIX, |
| SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19) |
| |
| # FIXME: Perhas these two routines should be part of the Port instead? |
| BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') |
| |
| WEBKIT_BUG_PREFIX = 'webkit.org/b/' |
| CHROMIUM_BUG_PREFIX = 'crbug.com/' |
| V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' |
| MOJO_BUG_PREFIX = 'https://github.com/domokit/mojo/issues/' |
| NAMED_BUG_PREFIX = 'Bug(' |
| |
| MISSING_KEYWORD = 'Missing' |
| NEEDS_REBASELINE_KEYWORD = 'NeedsRebaseline' |
| NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline' |
| |
| class ParseError(Exception): |
| def __init__(self, warnings): |
| super(ParseError, self).__init__() |
| self.warnings = warnings |
| |
| def __str__(self): |
| return '\n'.join(map(str, self.warnings)) |
| |
| def __repr__(self): |
| return 'ParseError(warnings=%s)' % self.warnings |
| |
| |
| class TestExpectationParser(object): |
| """Provides parsing facilities for lines in the test_expectation.txt file.""" |
| |
| # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents. |
| REBASELINE_MODIFIER = 'rebaseline' |
| NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' |
| NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' |
| PASS_EXPECTATION = 'pass' |
| SKIP_MODIFIER = 'skip' |
| SLOW_MODIFIER = 'slow' |
| WONTFIX_MODIFIER = 'wontfix' |
| |
| TIMEOUT_EXPECTATION = 'timeout' |
| |
| MISSING_BUG_WARNING = 'Test lacks BUG specifier.' |
| |
| def __init__(self, port, full_test_list, is_lint_mode): |
| self._port = port |
| self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) |
| self._full_test_list = full_test_list |
| self._is_lint_mode = is_lint_mode |
| |
| def parse(self, filename, expectations_string): |
| expectation_lines = [] |
| line_number = 0 |
| for line in expectations_string.split("\n"): |
| line_number += 1 |
| test_expectation = self._tokenize_line(filename, line, line_number) |
| self._parse_line(test_expectation) |
| expectation_lines.append(test_expectation) |
| return expectation_lines |
| |
| def _create_expectation_line(self, test_name, expectations, file_name): |
| expectation_line = TestExpectationLine() |
| expectation_line.original_string = test_name |
| expectation_line.name = test_name |
| expectation_line.filename = file_name |
| expectation_line.expectations = expectations |
| return expectation_line |
| |
| def expectation_line_for_test(self, test_name, expectations): |
| expectation_line = self._create_expectation_line(test_name, expectations, '<Bot TestExpectations>') |
| self._parse_line(expectation_line) |
| return expectation_line |
| |
| |
| def expectation_for_skipped_test(self, test_name): |
| if not self._port.test_exists(test_name): |
| _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name) |
| expectation_line = self._create_expectation_line(test_name, [TestExpectationParser.PASS_EXPECTATION], '<Skipped file>') |
| expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER] |
| expectation_line.is_skipped_outside_expectations_file = True |
| self._parse_line(expectation_line) |
| return expectation_line |
| |
| def _parse_line(self, expectation_line): |
| if not expectation_line.name: |
| return |
| |
| if not self._check_test_exists(expectation_line): |
| return |
| |
| expectation_line.is_file = self._port.test_isfile(expectation_line.name) |
| if expectation_line.is_file: |
| expectation_line.path = expectation_line.name |
| else: |
| expectation_line.path = self._port.normalize_test_name(expectation_line.name) |
| |
| self._collect_matching_tests(expectation_line) |
| |
| self._parse_specifiers(expectation_line) |
| self._parse_expectations(expectation_line) |
| |
| def _parse_specifiers(self, expectation_line): |
| if self._is_lint_mode: |
| self._lint_line(expectation_line) |
| |
| parsed_specifiers = set([specifier.lower() for specifier in expectation_line.specifiers]) |
| expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) |
| |
| def _lint_line(self, expectation_line): |
| expectations = [expectation.lower() for expectation in expectation_line.expectations] |
| if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectations: |
| expectation_line.warnings.append(self.MISSING_BUG_WARNING) |
| if self.REBASELINE_MODIFIER in expectations: |
| expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') |
| |
| if self.NEEDS_REBASELINE_MODIFIER in expectations or self.NEEDS_MANUAL_REBASELINE_MODIFIER in expectations: |
| for test in expectation_line.matching_tests: |
| if self._port.reference_files(test): |
| expectation_line.warnings.append('A reftest cannot be marked as NeedsRebaseline/NeedsManualRebaseline') |
| |
| def _parse_expectations(self, expectation_line): |
| result = set() |
| for part in expectation_line.expectations: |
| expectation = TestExpectations.expectation_from_string(part) |
| if expectation is None: # Careful, PASS is currently 0. |
| expectation_line.warnings.append('Unsupported expectation: %s' % part) |
| continue |
| result.add(expectation) |
| expectation_line.parsed_expectations = result |
| |
| def _check_test_exists(self, expectation_line): |
| # WebKit's way of skipping tests is to add a -disabled suffix. |
| # So we should consider the path existing if the path or the |
| # -disabled version exists. |
| if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): |
| # Log a warning here since you hit this case any |
| # time you update TestExpectations without syncing |
| # the tests directory |
| expectation_line.warnings.append('Path does not exist.') |
| return False |
| return True |
| |
| def _collect_matching_tests(self, expectation_line): |
| """Convert the test specification to an absolute, normalized |
| path and make sure directories end with the OS path separator.""" |
| # FIXME: full_test_list can quickly contain a big amount of |
| # elements. We should consider at some point to use a more |
| # efficient structure instead of a list. Maybe a dictionary of |
| # lists to represent the tree of tests, leaves being test |
| # files and nodes being categories. |
| |
| if not self._full_test_list: |
| expectation_line.matching_tests = [expectation_line.path] |
| return |
| |
| if not expectation_line.is_file: |
| # this is a test category, return all the tests of the category. |
| expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] |
| return |
| |
| # this is a test file, do a quick check if it's in the |
| # full test suite. |
| if expectation_line.path in self._full_test_list: |
| expectation_line.matching_tests.append(expectation_line.path) |
| |
| # FIXME: Update the original specifiers and remove this once the old syntax is gone. |
| _configuration_tokens_list = [ |
| 'Mac', 'SnowLeopard', 'Lion', 'Retina', 'MountainLion', 'Mavericks', |
| 'Win', 'XP', 'Win7', |
| 'Linux', |
| 'Android', |
| 'Release', |
| 'Debug', |
| ] |
| |
| _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) |
| _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) |
| |
| # FIXME: Update the original specifiers list and remove this once the old syntax is gone. |
| _expectation_tokens = { |
| 'Crash': 'CRASH', |
| 'Leak': 'LEAK', |
| 'Failure': 'FAIL', |
| 'ImageOnlyFailure': 'IMAGE', |
| MISSING_KEYWORD: 'MISSING', |
| 'Pass': 'PASS', |
| 'Rebaseline': 'REBASELINE', |
| NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', |
| NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', |
| 'Skip': 'SKIP', |
| 'Slow': 'SLOW', |
| 'Timeout': 'TIMEOUT', |
| 'WontFix': 'WONTFIX', |
| } |
| |
| _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] + |
| [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) |
| |
| # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. |
| @classmethod |
| def _tokenize_line(cls, filename, expectation_string, line_number): |
| """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. |
| |
| The new format for a test expectation line is: |
| |
| [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] |
| |
| Any errant whitespace is not preserved. |
| |
| """ |
| expectation_line = TestExpectationLine() |
| expectation_line.original_string = expectation_string |
| expectation_line.filename = filename |
| expectation_line.line_numbers = str(line_number) |
| |
| comment_index = expectation_string.find("#") |
| if comment_index == -1: |
| comment_index = len(expectation_string) |
| else: |
| expectation_line.comment = expectation_string[comment_index + 1:] |
| |
| remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) |
| if len(remaining_string) == 0: |
| return expectation_line |
| |
| # special-case parsing this so that we fail immediately instead of treating this as a test name |
| if remaining_string.startswith('//'): |
| expectation_line.warnings = ['use "#" instead of "//" for comments'] |
| return expectation_line |
| |
| bugs = [] |
| specifiers = [] |
| name = None |
| expectations = [] |
| warnings = [] |
| has_unrecognized_expectation = False |
| |
| tokens = remaining_string.split() |
| state = 'start' |
| for token in tokens: |
| if (token.startswith(WEBKIT_BUG_PREFIX) or |
| token.startswith(CHROMIUM_BUG_PREFIX) or |
| token.startswith(V8_BUG_PREFIX) or |
| token.startswith(MOJO_BUG_PREFIX) or |
| token.startswith(NAMED_BUG_PREFIX)): |
| if state != 'start': |
| warnings.append('"%s" is not at the start of the line.' % token) |
| break |
| if (token.startswith(WEBKIT_BUG_PREFIX) or |
| token.startswith(CHROMIUM_BUG_PREFIX) or |
| token.startswith(MOJO_BUG_PREFIX) or |
| token.startswith(V8_BUG_PREFIX)): |
| bugs.append(token) |
| else: |
| match = re.match('Bug\((\w+)\)$', token) |
| if not match: |
| warnings.append('unrecognized bug identifier "%s"' % token) |
| break |
| else: |
| bugs.append(token) |
| elif token == '[': |
| if state == 'start': |
| state = 'configuration' |
| elif state == 'name_found': |
| state = 'expectations' |
| else: |
| warnings.append('unexpected "["') |
| break |
| elif token == ']': |
| if state == 'configuration': |
| state = 'name' |
| elif state == 'expectations': |
| state = 'done' |
| else: |
| warnings.append('unexpected "]"') |
| break |
| elif token in ('//', ':', '='): |
| warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) |
| break |
| elif state == 'configuration': |
| specifiers.append(cls._configuration_tokens.get(token, token)) |
| elif state == 'expectations': |
| if token not in cls._expectation_tokens: |
| has_unrecognized_expectation = True |
| warnings.append('Unrecognized expectation "%s"' % token) |
| else: |
| expectations.append(cls._expectation_tokens.get(token, token)) |
| elif state == 'name_found': |
| warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) |
| break |
| else: |
| name = token |
| state = 'name_found' |
| |
| if not warnings: |
| if not name: |
| warnings.append('Did not find a test name.') |
| elif state not in ('name_found', 'done'): |
| warnings.append('Missing a "]"') |
| |
| if 'WONTFIX' in expectations and 'SKIP' not in expectations: |
| expectations.append('SKIP') |
| |
| if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])): |
| warnings.append('A test marked Skip or WontFix must not have other expectations.') |
| |
| if not expectations and not has_unrecognized_expectation: |
| warnings.append('Missing expectations.') |
| |
| expectation_line.bugs = bugs |
| expectation_line.specifiers = specifiers |
| expectation_line.expectations = expectations |
| expectation_line.name = name |
| expectation_line.warnings = warnings |
| return expectation_line |
| |
| @classmethod |
| def _split_space_separated(cls, space_separated_string): |
| """Splits a space-separated string into an array.""" |
| return [part.strip() for part in space_separated_string.strip().split(' ')] |
| |
| |
| class TestExpectationLine(object): |
| """Represents a line in test expectations file.""" |
| |
| def __init__(self): |
| """Initializes a blank-line equivalent of an expectation.""" |
| self.original_string = None |
| self.filename = None # this is the path to the expectations file for this line |
| self.line_numbers = "0" |
| self.name = None # this is the path in the line itself |
| self.path = None # this is the normpath of self.name |
| self.bugs = [] |
| self.specifiers = [] |
| self.parsed_specifiers = [] |
| self.matching_configurations = set() |
| self.expectations = [] |
| self.parsed_expectations = set() |
| self.comment = None |
| self.matching_tests = [] |
| self.warnings = [] |
| self.is_skipped_outside_expectations_file = False |
| |
| def __eq__(self, other): |
| return (self.original_string == other.original_string |
| and self.filename == other.filename |
| and self.line_numbers == other.line_numbers |
| and self.name == other.name |
| and self.path == other.path |
| and self.bugs == other.bugs |
| and self.specifiers == other.specifiers |
| and self.parsed_specifiers == other.parsed_specifiers |
| and self.matching_configurations == other.matching_configurations |
| and self.expectations == other.expectations |
| and self.parsed_expectations == other.parsed_expectations |
| and self.comment == other.comment |
| and self.matching_tests == other.matching_tests |
| and self.warnings == other.warnings |
| and self.is_skipped_outside_expectations_file == other.is_skipped_outside_expectations_file) |
| |
| def is_invalid(self): |
| return bool(self.warnings and self.warnings != [TestExpectationParser.MISSING_BUG_WARNING]) |
| |
| def is_flaky(self): |
| return len(self.parsed_expectations) > 1 |
| |
| def is_whitespace_or_comment(self): |
| return bool(re.match("^\s*$", self.original_string.split('#')[0])) |
| |
| @staticmethod |
| def create_passing_expectation(test): |
| expectation_line = TestExpectationLine() |
| expectation_line.name = test |
| expectation_line.path = test |
| expectation_line.parsed_expectations = set([PASS]) |
| expectation_line.expectations = set(['PASS']) |
| expectation_line.matching_tests = [test] |
| return expectation_line |
| |
| @staticmethod |
| def merge_expectation_lines(line1, line2, model_all_expectations): |
| """Merges the expectations of line2 into line1 and returns a fresh object.""" |
| if line1 is None: |
| return line2 |
| if line2 is None: |
| return line1 |
| if model_all_expectations and line1.filename != line2.filename: |
| return line2 |
| |
| # Don't merge original_string or comment. |
| result = TestExpectationLine() |
| # We only care about filenames when we're linting, in which case the filenames are the same. |
| # Not clear that there's anything better to do when not linting and the filenames are different. |
| if model_all_expectations: |
| result.filename = line2.filename |
| result.line_numbers = line1.line_numbers + "," + line2.line_numbers |
| result.name = line1.name |
| result.path = line1.path |
| result.parsed_expectations = set(line1.parsed_expectations) | set(line2.parsed_expectations) |
| result.expectations = list(set(line1.expectations) | set(line2.expectations)) |
| result.bugs = list(set(line1.bugs) | set(line2.bugs)) |
| result.specifiers = list(set(line1.specifiers) | set(line2.specifiers)) |
| result.parsed_specifiers = list(set(line1.parsed_specifiers) | set(line2.parsed_specifiers)) |
| result.matching_configurations = set(line1.matching_configurations) | set(line2.matching_configurations) |
| result.matching_tests = list(list(set(line1.matching_tests) | set(line2.matching_tests))) |
| result.warnings = list(set(line1.warnings) | set(line2.warnings)) |
| result.is_skipped_outside_expectations_file = line1.is_skipped_outside_expectations_file or line2.is_skipped_outside_expectations_file |
| return result |
| |
| def to_string(self, test_configuration_converter, include_specifiers=True, include_expectations=True, include_comment=True): |
| parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()]) |
| |
| if self.is_invalid(): |
| return self.original_string or '' |
| |
| if self.name is None: |
| return '' if self.comment is None else "#%s" % self.comment |
| |
| if test_configuration_converter and self.bugs: |
| specifiers_list = test_configuration_converter.to_specifiers_list(self.matching_configurations) |
| result = [] |
| for specifiers in specifiers_list: |
| # FIXME: this is silly that we join the specifiers and then immediately split them. |
| specifiers = self._serialize_parsed_specifiers(test_configuration_converter, specifiers).split() |
| expectations = self._serialize_parsed_expectations(parsed_expectation_to_string).split() |
| result.append(self._format_line(self.bugs, specifiers, self.name, expectations, self.comment)) |
| return "\n".join(result) if result else None |
| |
| return self._format_line(self.bugs, self.specifiers, self.name, self.expectations, self.comment, |
| include_specifiers, include_expectations, include_comment) |
| |
| def to_csv(self): |
| # Note that this doesn't include the comments. |
| return '%s,%s,%s,%s' % (self.name, ' '.join(self.bugs), ' '.join(self.specifiers), ' '.join(self.expectations)) |
| |
| def _serialize_parsed_expectations(self, parsed_expectation_to_string): |
| result = [] |
| for index in TestExpectations.EXPECTATIONS.values(): |
| if index in self.parsed_expectations: |
| result.append(parsed_expectation_to_string[index]) |
| return ' '.join(result) |
| |
| def _serialize_parsed_specifiers(self, test_configuration_converter, specifiers): |
| result = [] |
| result.extend(sorted(self.parsed_specifiers)) |
| result.extend(test_configuration_converter.specifier_sorter().sort_specifiers(specifiers)) |
| return ' '.join(result) |
| |
| @staticmethod |
| def _filter_redundant_expectations(expectations): |
| if set(expectations) == set(['Pass', 'Skip']): |
| return ['Skip'] |
| if set(expectations) == set(['Pass', 'Slow']): |
| return ['Slow'] |
| return expectations |
| |
| @staticmethod |
| def _format_line(bugs, specifiers, name, expectations, comment, include_specifiers=True, include_expectations=True, include_comment=True): |
| new_specifiers = [] |
| new_expectations = [] |
| for specifier in specifiers: |
| # FIXME: Make this all work with the mixed-cased specifiers (e.g. WontFix, Slow, etc). |
| specifier = specifier.upper() |
| new_specifiers.append(TestExpectationParser._inverted_configuration_tokens.get(specifier, specifier)) |
| |
| for expectation in expectations: |
| expectation = expectation.upper() |
| new_expectations.append(TestExpectationParser._inverted_expectation_tokens.get(expectation, expectation)) |
| |
| result = '' |
| if include_specifiers and (bugs or new_specifiers): |
| if bugs: |
| result += ' '.join(bugs) + ' ' |
| if new_specifiers: |
| result += '[ %s ] ' % ' '.join(new_specifiers) |
| result += name |
| if include_expectations and new_expectations: |
| new_expectations = TestExpectationLine._filter_redundant_expectations(new_expectations) |
| result += ' [ %s ]' % ' '.join(sorted(set(new_expectations))) |
| if include_comment and comment is not None: |
| result += " #%s" % comment |
| return result |
| |
| |
| # FIXME: Refactor API to be a proper CRUD. |
| class TestExpectationsModel(object): |
| """Represents relational store of all expectations and provides CRUD semantics to manage it.""" |
| |
| def __init__(self, shorten_filename=None): |
| # Maps a test to its list of expectations. |
| self._test_to_expectations = {} |
| |
| # Maps a test to list of its specifiers (string values) |
| self._test_to_specifiers = {} |
| |
| # Maps a test to a TestExpectationLine instance. |
| self._test_to_expectation_line = {} |
| |
| self._expectation_to_tests = self._dict_of_sets(TestExpectations.EXPECTATIONS) |
| self._timeline_to_tests = self._dict_of_sets(TestExpectations.TIMELINES) |
| self._result_type_to_tests = self._dict_of_sets(TestExpectations.RESULT_TYPES) |
| |
| self._shorten_filename = shorten_filename or (lambda x: x) |
| |
| def _merge_test_map(self, self_map, other_map): |
| for test in other_map: |
| new_expectations = set(other_map[test]) |
| if test in self_map: |
| new_expectations |= set(self_map[test]) |
| self_map[test] = list(new_expectations) if isinstance(other_map[test], list) else new_expectations |
| |
| def _merge_dict_of_sets(self, self_dict, other_dict): |
| for key in other_dict: |
| self_dict[key] |= other_dict[key] |
| |
| def merge_model(self, other): |
| self._merge_test_map(self._test_to_expectations, other._test_to_expectations) |
| |
| for test, line in other._test_to_expectation_line.items(): |
| if test in self._test_to_expectation_line: |
| line = TestExpectationLine.merge_expectation_lines(self._test_to_expectation_line[test], line, model_all_expectations=False) |
| self._test_to_expectation_line[test] = line |
| |
| self._merge_dict_of_sets(self._expectation_to_tests, other._expectation_to_tests) |
| self._merge_dict_of_sets(self._timeline_to_tests, other._timeline_to_tests) |
| self._merge_dict_of_sets(self._result_type_to_tests, other._result_type_to_tests) |
| |
| def _dict_of_sets(self, strings_to_constants): |
| """Takes a dict of strings->constants and returns a dict mapping |
| each constant to an empty set.""" |
| d = {} |
| for c in strings_to_constants.values(): |
| d[c] = set() |
| return d |
| |
| def get_test_set(self, expectation, include_skips=True): |
| tests = self._expectation_to_tests[expectation] |
| if not include_skips: |
| tests = tests - self.get_test_set(SKIP) |
| return tests |
| |
| def get_test_set_for_keyword(self, keyword): |
| expectation_enum = TestExpectations.EXPECTATIONS.get(keyword.lower(), None) |
| if expectation_enum is not None: |
| return self._expectation_to_tests[expectation_enum] |
| |
| matching_tests = set() |
| for test, specifiers in self._test_to_specifiers.iteritems(): |
| if keyword.lower() in specifiers: |
| matching_tests.add(test) |
| return matching_tests |
| |
| def get_tests_with_result_type(self, result_type): |
| return self._result_type_to_tests[result_type] |
| |
| def get_tests_with_timeline(self, timeline): |
| return self._timeline_to_tests[timeline] |
| |
| def has_test(self, test): |
| return test in self._test_to_expectation_line |
| |
| def get_expectation_line(self, test): |
| return self._test_to_expectation_line.get(test) |
| |
| def get_expectations(self, test): |
| return self._test_to_expectations[test] |
| |
| def get_expectations_string(self, test): |
| """Returns the expectatons for the given test as an uppercase string. |
| If there are no expectations for the test, then "PASS" is returned.""" |
| if self.get_expectation_line(test).is_skipped_outside_expectations_file: |
| return 'NOTRUN' |
| |
| expectations = self.get_expectations(test) |
| retval = [] |
| |
| # FIXME: WontFix should cause the test to get skipped without artificially adding SKIP to the expectations list. |
| if WONTFIX in expectations and SKIP in expectations: |
| expectations.remove(SKIP) |
| |
| for expectation in expectations: |
| retval.append(self.expectation_to_string(expectation)) |
| |
| return " ".join(retval) |
| |
| def expectation_to_string(self, expectation): |
| """Return the uppercased string equivalent of a given expectation.""" |
| for item in TestExpectations.EXPECTATIONS.items(): |
| if item[1] == expectation: |
| return item[0].upper() |
| raise ValueError(expectation) |
| |
| def remove_expectation_line(self, test): |
| if not self.has_test(test): |
| return |
| self._clear_expectations_for_test(test) |
| del self._test_to_expectation_line[test] |
| |
| def add_expectation_line(self, expectation_line, |
| model_all_expectations=False): |
| """Returns a list of warnings encountered while matching specifiers.""" |
| |
| if expectation_line.is_invalid(): |
| return |
| |
| for test in expectation_line.matching_tests: |
| if self._already_seen_better_match(test, expectation_line): |
| continue |
| |
| if model_all_expectations: |
| expectation_line = TestExpectationLine.merge_expectation_lines(self.get_expectation_line(test), expectation_line, model_all_expectations) |
| |
| self._clear_expectations_for_test(test) |
| self._test_to_expectation_line[test] = expectation_line |
| self._add_test(test, expectation_line) |
| |
| def _add_test(self, test, expectation_line): |
| """Sets the expected state for a given test. |
| |
| This routine assumes the test has not been added before. If it has, |
| use _clear_expectations_for_test() to reset the state prior to |
| calling this.""" |
| self._test_to_expectations[test] = expectation_line.parsed_expectations |
| for expectation in expectation_line.parsed_expectations: |
| self._expectation_to_tests[expectation].add(test) |
| |
| self._test_to_specifiers[test] = expectation_line.specifiers |
| |
| if WONTFIX in expectation_line.parsed_expectations: |
| self._timeline_to_tests[WONTFIX].add(test) |
| else: |
| self._timeline_to_tests[NOW].add(test) |
| |
| if SKIP in expectation_line.parsed_expectations: |
| self._result_type_to_tests[SKIP].add(test) |
| elif expectation_line.parsed_expectations == set([PASS]): |
| self._result_type_to_tests[PASS].add(test) |
| elif expectation_line.is_flaky(): |
| self._result_type_to_tests[FLAKY].add(test) |
| else: |
| # FIXME: What is this? |
| self._result_type_to_tests[FAIL].add(test) |
| |
| def _clear_expectations_for_test(self, test): |
| """Remove prexisting expectations for this test. |
| This happens if we are seeing a more precise path |
| than a previous listing. |
| """ |
| if self.has_test(test): |
| self._test_to_expectations.pop(test, '') |
| self._remove_from_sets(test, self._expectation_to_tests) |
| self._remove_from_sets(test, self._timeline_to_tests) |
| self._remove_from_sets(test, self._result_type_to_tests) |
| |
| def _remove_from_sets(self, test, dict_of_sets_of_tests): |
| """Removes the given test from the sets in the dictionary. |
| |
| Args: |
| test: test to look for |
| dict: dict of sets of files""" |
| for set_of_tests in dict_of_sets_of_tests.itervalues(): |
| if test in set_of_tests: |
| set_of_tests.remove(test) |
| |
| def _already_seen_better_match(self, test, expectation_line): |
| """Returns whether we've seen a better match already in the file. |
| |
| Returns True if we've already seen a expectation_line.name that matches more of the test |
| than this path does |
| """ |
| # FIXME: See comment below about matching test configs and specificity. |
| if not self.has_test(test): |
| # We've never seen this test before. |
| return False |
| |
| prev_expectation_line = self._test_to_expectation_line[test] |
| |
| if prev_expectation_line.filename != expectation_line.filename: |
| # We've moved on to a new expectation file, which overrides older ones. |
| return False |
| |
| if len(prev_expectation_line.path) > len(expectation_line.path): |
| # The previous path matched more of the test. |
| return True |
| |
| if len(prev_expectation_line.path) < len(expectation_line.path): |
| # This path matches more of the test. |
| return False |
| |
| # At this point we know we have seen a previous exact match on this |
| # base path, so we need to check the two sets of specifiers. |
| |
| # FIXME: This code was originally designed to allow lines that matched |
| # more specifiers to override lines that matched fewer specifiers. |
| # However, we currently view these as errors. |
| # |
| # To use the "more specifiers wins" policy, change the errors for overrides |
| # to be warnings and return False". |
| |
| if prev_expectation_line.matching_configurations == expectation_line.matching_configurations: |
| expectation_line.warnings.append('Duplicate or ambiguous entry lines %s:%s and %s:%s.' % ( |
| self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, |
| self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) |
| return True |
| |
| if prev_expectation_line.matching_configurations >= expectation_line.matching_configurations: |
| expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, |
| self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, |
| self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) |
| # FIXME: return False if we want more specific to win. |
| return True |
| |
| if prev_expectation_line.matching_configurations <= expectation_line.matching_configurations: |
| expectation_line.warnings.append('More specific entry for %s on line %s:%s overrides line %s:%s.' % (expectation_line.name, |
| self._shorten_filename(expectation_line.filename), expectation_line.line_numbers, |
| self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers)) |
| return True |
| |
| if prev_expectation_line.matching_configurations & expectation_line.matching_configurations: |
| expectation_line.warnings.append('Entries for %s on lines %s:%s and %s:%s match overlapping sets of configurations.' % (expectation_line.name, |
| self._shorten_filename(prev_expectation_line.filename), prev_expectation_line.line_numbers, |
| self._shorten_filename(expectation_line.filename), expectation_line.line_numbers)) |
| return True |
| |
| # Configuration sets are disjoint, then. |
| return False |
| |
| |
| class TestExpectations(object): |
| """Test expectations consist of lines with specifications of what |
| to expect from layout test cases. The test cases can be directories |
| in which case the expectations apply to all test cases in that |
| directory and any subdirectory. The format is along the lines of: |
| |
| tests/fast/js/fixme.js [ Failure ] |
| tests/fast/js/flaky.js [ Failure Pass ] |
| tests/fast/js/crash.js [ Crash Failure Pass Timeout ] |
| ... |
| |
| To add specifiers: |
| tests/fast/js/no-good.js |
| [ Debug ] tests/fast/js/no-good.js [ Pass Timeout ] |
| [ Debug ] tests/fast/js/no-good.js [ Pass Skip Timeout ] |
| [ Linux Debug ] tests/fast/js/no-good.js [ Pass Skip Timeout ] |
| [ Linux Win ] tests/fast/js/no-good.js [ Pass Skip Timeout ] |
| |
| Skip: Doesn't run the test. |
| Slow: The test takes a long time to run, but does not timeout indefinitely. |
| WontFix: For tests that we never intend to pass on a given platform (treated like Skip). |
| |
| Notes: |
| -A test cannot be both SLOW and TIMEOUT |
| -A test can be included twice, but not via the same path. |
| -If a test is included twice, then the more precise path wins. |
| -CRASH tests cannot be WONTFIX |
| """ |
| |
| # FIXME: Update to new syntax once the old format is no longer supported. |
| EXPECTATIONS = {'pass': PASS, |
| 'audio': AUDIO, |
| 'fail': FAIL, |
| 'image': IMAGE, |
| 'image+text': IMAGE_PLUS_TEXT, |
| 'text': TEXT, |
| 'timeout': TIMEOUT, |
| 'crash': CRASH, |
| 'leak': LEAK, |
| 'missing': MISSING, |
| TestExpectationParser.SKIP_MODIFIER: SKIP, |
| TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBASELINE, |
| TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEEDS_MANUAL_REBASELINE, |
| TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
| TestExpectationParser.SLOW_MODIFIER: SLOW, |
| TestExpectationParser.REBASELINE_MODIFIER: REBASELINE, |
| } |
| |
| EXPECTATIONS_TO_STRING = dict((k, v) for (v, k) in EXPECTATIONS.iteritems()) |
| |
| # (aggregated by category, pass/fail/skip, type) |
| EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped', |
| PASS: 'passes', |
| FAIL: 'failures', |
| IMAGE: 'image-only failures', |
| TEXT: 'text-only failures', |
| IMAGE_PLUS_TEXT: 'image and text failures', |
| AUDIO: 'audio failures', |
| CRASH: 'crashes', |
| LEAK: 'leaks', |
| TIMEOUT: 'timeouts', |
| MISSING: 'missing results'} |
| |
| NON_TEST_OUTCOME_EXPECTATIONS = (REBASELINE, SKIP, SLOW, WONTFIX) |
| |
| BUILD_TYPES = ('debug', 'release') |
| |
| TIMELINES = {TestExpectationParser.WONTFIX_MODIFIER: WONTFIX, |
| 'now': NOW} |
| |
| RESULT_TYPES = {'skip': SKIP, |
| 'pass': PASS, |
| 'fail': FAIL, |
| 'flaky': FLAKY} |
| |
| @classmethod |
| def expectation_from_string(cls, string): |
| assert(' ' not in string) # This only handles one expectation at a time. |
| return cls.EXPECTATIONS.get(string.lower()) |
| |
| @staticmethod |
| def result_was_expected(result, expected_results, test_needs_rebaselining): |
| """Returns whether we got a result we were expecting. |
| Args: |
| result: actual result of a test execution |
| expected_results: set of results listed in test_expectations |
| test_needs_rebaselining: whether test was marked as REBASELINE""" |
| if not (set(expected_results) - (set(TestExpectations.NON_TEST_OUTCOME_EXPECTATIONS))): |
| expected_results = set([PASS]) |
| |
| if result in expected_results: |
| return True |
| if result in (PASS, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, MISSING) and (NEEDS_REBASELINE in expected_results or NEEDS_MANUAL_REBASELINE in expected_results): |
| return True |
| if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results): |
| return True |
| if result == MISSING and test_needs_rebaselining: |
| return True |
| if result == SKIP: |
| return True |
| return False |
| |
| @staticmethod |
| def remove_pixel_failures(expected_results): |
| """Returns a copy of the expected results for a test, except that we |
| drop any pixel failures and return the remaining expectations. For example, |
| if we're not running pixel tests, then tests expected to fail as IMAGE |
| will PASS.""" |
| expected_results = expected_results.copy() |
| if IMAGE in expected_results: |
| expected_results.remove(IMAGE) |
| expected_results.add(PASS) |
| return expected_results |
| |
| @staticmethod |
| def remove_non_sanitizer_failures(expected_results): |
| """Returns a copy of the expected results for a test, except that we |
| drop any failures that the sanitizers don't care about.""" |
| expected_results = expected_results.copy() |
| for result in (IMAGE, FAIL, IMAGE_PLUS_TEXT): |
| if result in expected_results: |
| expected_results.remove(result) |
| expected_results.add(PASS) |
| return expected_results |
| |
| @staticmethod |
| def has_pixel_failures(actual_results): |
| return IMAGE in actual_results or FAIL in actual_results |
| |
| @staticmethod |
| def suffixes_for_expectations(expectations): |
| suffixes = set() |
| if IMAGE in expectations: |
| suffixes.add('png') |
| if FAIL in expectations: |
| suffixes.add('txt') |
| suffixes.add('png') |
| suffixes.add('wav') |
| return set(suffixes) |
| |
| @staticmethod |
| def suffixes_for_actual_expectations_string(expectations): |
| suffixes = set() |
| if 'TEXT' in expectations: |
| suffixes.add('txt') |
| if 'IMAGE' in expectations: |
| suffixes.add('png') |
| if 'AUDIO' in expectations: |
| suffixes.add('wav') |
| if 'MISSING' in expectations: |
| suffixes.add('txt') |
| suffixes.add('png') |
| suffixes.add('wav') |
| return suffixes |
| |
| # FIXME: This constructor does too much work. We should move the actual parsing of |
| # the expectations into separate routines so that linting and handling overrides |
| # can be controlled separately, and the constructor can be more of a no-op. |
| def __init__(self, port, tests=None, include_overrides=True, expectations_dict=None, model_all_expectations=False, is_lint_mode=False): |
| self._full_test_list = tests |
| self._test_config = port.test_configuration() |
| self._is_lint_mode = is_lint_mode |
| self._model_all_expectations = self._is_lint_mode or model_all_expectations |
| self._model = TestExpectationsModel(self._shorten_filename) |
| self._parser = TestExpectationParser(port, tests, self._is_lint_mode) |
| self._port = port |
| self._skipped_tests_warnings = [] |
| self._expectations = [] |
| |
| if not expectations_dict: |
| expectations_dict = port.expectations_dict() |
| |
| # Always parse the generic expectations (the generic file is required |
| # to be the first one in the expectations_dict, which must be an OrderedDict). |
| generic_path, generic_exps = expectations_dict.items()[0] |
| expectations = self._parser.parse(generic_path, generic_exps) |
| self._add_expectations(expectations, self._model) |
| self._expectations += expectations |
| |
| # Now add the overrides if so requested. |
| if include_overrides: |
| for path, contents in expectations_dict.items()[1:]: |
| expectations = self._parser.parse(path, contents) |
| model = TestExpectationsModel(self._shorten_filename) |
| self._add_expectations(expectations, model) |
| self._expectations += expectations |
| self._model.merge_model(model) |
| |
| # FIXME: move ignore_tests into port.skipped_layout_tests() |
| self.add_extra_skipped_tests(port.skipped_layout_tests(tests).union(set(port.get_option('ignore_tests', [])))) |
| self.add_expectations_from_bot() |
| |
| self._has_warnings = False |
| self._report_warnings() |
| self._process_tests_without_expectations() |
| |
| # TODO(ojan): Allow for removing skipped tests when getting the list of |
| # tests to run, but not when getting metrics. |
| def model(self): |
| return self._model |
| |
| def get_needs_rebaseline_failures(self): |
| return self._model.get_test_set(NEEDS_REBASELINE) |
| |
| def get_rebaselining_failures(self): |
| return self._model.get_test_set(REBASELINE) |
| |
| # FIXME: Change the callsites to use TestExpectationsModel and remove. |
| def get_expectations(self, test): |
| return self._model.get_expectations(test) |
| |
| # FIXME: Change the callsites to use TestExpectationsModel and remove. |
| def get_tests_with_result_type(self, result_type): |
| return self._model.get_tests_with_result_type(result_type) |
| |
| # FIXME: Change the callsites to use TestExpectationsModel and remove. |
| def get_test_set(self, expectation, include_skips=True): |
| return self._model.get_test_set(expectation, include_skips) |
| |
| # FIXME: Change the callsites to use TestExpectationsModel and remove. |
| def get_tests_with_timeline(self, timeline): |
| return self._model.get_tests_with_timeline(timeline) |
| |
| def get_expectations_string(self, test): |
| return self._model.get_expectations_string(test) |
| |
| def expectation_to_string(self, expectation): |
| return self._model.expectation_to_string(expectation) |
| |
| def matches_an_expected_result(self, test, result, pixel_tests_are_enabled, sanitizer_is_enabled): |
| expected_results = self._model.get_expectations(test) |
| if sanitizer_is_enabled: |
| expected_results = self.remove_non_sanitizer_failures(expected_results) |
| elif not pixel_tests_are_enabled: |
| expected_results = self.remove_pixel_failures(expected_results) |
| return self.result_was_expected(result, expected_results, self.is_rebaselining(test)) |
| |
| def is_rebaselining(self, test): |
| return REBASELINE in self._model.get_expectations(test) |
| |
| def _shorten_filename(self, filename): |
| if filename.startswith(self._port.path_from_webkit_base()): |
| return self._port.host.filesystem.relpath(filename, self._port.path_from_webkit_base()) |
| return filename |
| |
| def _report_warnings(self): |
| warnings = [] |
| for expectation in self._expectations: |
| for warning in expectation.warnings: |
| warnings.append('%s:%s %s %s' % (self._shorten_filename(expectation.filename), expectation.line_numbers, |
| warning, expectation.name if expectation.expectations else expectation.original_string)) |
| |
| if warnings: |
| self._has_warnings = True |
| if self._is_lint_mode: |
| raise ParseError(warnings) |
| _log.warning('--lint-test-files warnings:') |
| for warning in warnings: |
| _log.warning(warning) |
| _log.warning('') |
| |
| def _process_tests_without_expectations(self): |
| if self._full_test_list: |
| for test in self._full_test_list: |
| if not self._model.has_test(test): |
| self._model.add_expectation_line(TestExpectationLine.create_passing_expectation(test)) |
| |
| def has_warnings(self): |
| return self._has_warnings |
| |
| def remove_configurations(self, removals): |
| expectations_to_remove = [] |
| modified_expectations = [] |
| |
| for test, test_configuration in removals: |
| for expectation in self._expectations: |
| if expectation.name != test or not expectation.parsed_expectations: |
| continue |
| if test_configuration not in expectation.matching_configurations: |
| continue |
| |
| expectation.matching_configurations.remove(test_configuration) |
| if expectation.matching_configurations: |
| modified_expectations.append(expectation) |
| else: |
| expectations_to_remove.append(expectation) |
| |
| for expectation in expectations_to_remove: |
| index = self._expectations.index(expectation) |
| self._expectations.remove(expectation) |
| |
| if index == len(self._expectations) or self._expectations[index].is_whitespace_or_comment(): |
| while index and self._expectations[index - 1].is_whitespace_or_comment(): |
| index = index - 1 |
| self._expectations.pop(index) |
| |
| return self.list_to_string(self._expectations, self._parser._test_configuration_converter, modified_expectations) |
| |
| def _add_expectations(self, expectation_list, model): |
| for expectation_line in expectation_list: |
| if not expectation_line.expectations: |
| continue |
| |
| if self._model_all_expectations or self._test_config in expectation_line.matching_configurations: |
| model.add_expectation_line(expectation_line, model_all_expectations=self._model_all_expectations) |
| |
| def add_extra_skipped_tests(self, tests_to_skip): |
| if not tests_to_skip: |
| return |
| for test in self._expectations: |
| if test.name and test.name in tests_to_skip: |
| test.warnings.append('%s:%s %s is also in a Skipped file.' % (test.filename, test.line_numbers, test.name)) |
| |
| model = TestExpectationsModel(self._shorten_filename) |
| for test_name in tests_to_skip: |
| expectation_line = self._parser.expectation_for_skipped_test(test_name) |
| model.add_expectation_line(expectation_line) |
| self._model.merge_model(model) |
| |
| def add_expectations_from_bot(self): |
| # FIXME: With mode 'very-flaky' and 'maybe-flaky', this will show the expectations entry in the flakiness |
| # dashboard rows for each test to be whatever the bot thinks they should be. Is this a good thing? |
| bot_expectations = self._port.bot_expectations() |
| model = TestExpectationsModel(self._shorten_filename) |
| for test_name in bot_expectations: |
| expectation_line = self._parser.expectation_line_for_test(test_name, bot_expectations[test_name]) |
| |
| # Unexpected results are merged into existing expectations. |
| merge = self._port.get_option('ignore_flaky_tests') == 'unexpected' |
| model.add_expectation_line(expectation_line) |
| self._model.merge_model(model) |
| |
| def add_expectation_line(self, expectation_line): |
| self._model.add_expectation_line(expectation_line) |
| self._expectations += [expectation_line] |
| |
| def remove_expectation_line(self, test): |
| if not self._model.has_test(test): |
| return |
| self._expectations.remove(self._model.get_expectation_line(test)) |
| self._model.remove_expectation_line(test) |
| |
| @staticmethod |
| def list_to_string(expectation_lines, test_configuration_converter=None, reconstitute_only_these=None): |
| def serialize(expectation_line): |
| # If reconstitute_only_these is an empty list, we want to return original_string. |
| # So we need to compare reconstitute_only_these to None, not just check if it's falsey. |
| if reconstitute_only_these is None or expectation_line in reconstitute_only_these: |
| return expectation_line.to_string(test_configuration_converter) |
| return expectation_line.original_string |
| |
| def nones_out(expectation_line): |
| return expectation_line is not None |
| |
| return "\n".join(filter(nones_out, map(serialize, expectation_lines))) |
