Hello community, here is the log from the commit of package python-hypothesis for openSUSE:Factory checked in at 2017-11-21 15:28:39 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-hypothesis (Old) and /work/SRC/openSUSE:Factory/.python-hypothesis.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-hypothesis" Tue Nov 21 15:28:39 2017 rev:11 rq:542943 version:3.38.0 Changes: -------- --- /work/SRC/openSUSE:Factory/python-hypothesis/python-hypothesis.changes 2017-11-17 10:37:15.294681272 +0100 +++ /work/SRC/openSUSE:Factory/.python-hypothesis.new/python-hypothesis.changes 2017-11-21 15:28:52.588811435 +0100 @@ -1,0 +2,28 @@ +Sun Nov 19 05:19:56 UTC 2017 - a...@gmx.de + +- update to version 3.38.0: + * This release overhauls the health check system in a variety of + small ways. It adds no new features, but is nevertheless a minor + release because it changes which tests are likely to fail health + checks. + * The most noticeable effect is that some tests that used to fail + health checks will now pass, and some that used to pass will + fail. These should all be improvements in accuracy. In particular: + + New failures will usually be because they are now taking into + account things like use of data() and assume() inside the test + body. + + New failures may also be because for some classes of example the + way data generation performance was measured was artificially + faster than real data generation (for most examples that are + hitting performance health checks the opposite should be the + case). + + Tests that used to fail health checks and now pass do so because + the health check system used to run in a way that was subtly + different than the main Hypothesis data generation and lacked + some of its support for e.g. large examples. + * If your data generation is especially slow, you may also see your + tests get somewhat faster, as there is no longer a separate health + check phase. This will be particularly noticeable when rerunning + test failures. + +------------------------------------------------------------------- Old: ---- hypothesis-3.37.0.tar.gz New: ---- hypothesis-3.38.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-hypothesis.spec ++++++ --- /var/tmp/diff_new_pack.afG22Z/_old 2017-11-21 15:28:54.116756057 +0100 +++ /var/tmp/diff_new_pack.afG22Z/_new 2017-11-21 15:28:54.116756057 +0100 @@ -24,7 +24,7 @@ %endif %bcond_with test Name: python-hypothesis -Version: 3.37.0 +Version: 3.38.0 Release: 0 Summary: A library for property based testing License: MPL-2.0 ++++++ hypothesis-3.37.0.tar.gz -> hypothesis-3.38.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/PKG-INFO new/hypothesis-3.38.0/PKG-INFO --- old/hypothesis-3.37.0/PKG-INFO 2017-11-12 21:11:25.000000000 +0100 +++ new/hypothesis-3.38.0/PKG-INFO 2017-11-18 11:27:46.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: hypothesis -Version: 3.37.0 +Version: 3.38.0 Summary: A library for property based testing Home-page: https://github.com/HypothesisWorks/hypothesis-python Author: David R. MacIver diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/core.py new/hypothesis-3.38.0/src/hypothesis/core.py --- old/hypothesis-3.37.0/src/hypothesis/core.py 2017-11-12 21:10:23.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/core.py 2017-11-18 11:26:22.000000000 +0100 @@ -54,11 +54,12 @@ arg_string, impersonate, function_digest, fully_qualified_name, \ define_function_signature, convert_positional_arguments, \ get_pretty_function_description +from hypothesis.internal.healthcheck import fail_health_check from hypothesis.internal.conjecture.data import Status, StopTest, \ ConjectureData from hypothesis.searchstrategy.strategies import SearchStrategy from hypothesis.internal.conjecture.engine import ExitReason, \ - ConjectureRunner, uniform, sort_key + ConjectureRunner, sort_key try: from coverage.tracer import CFileDisposition as FileDisposition @@ -278,119 +279,6 @@ raise -def fail_health_check(settings, message, label): - # Tell pytest to omit the body of this function from tracebacks - # http://doc.pytest.org/en/latest/example/simple.html#writing-well-integrated-assertion-helpers - __tracebackhide__ = True - - if label in settings.suppress_health_check: - return - if not settings.perform_health_check: - return - message += ( - '\nSee https://hypothesis.readthedocs.io/en/latest/health' - 'checks.html for more information about this. ' - ) - message += ( - 'If you want to disable just this health check, add %s ' - 'to the suppress_health_check settings for this test.' - ) % (label,) - raise FailedHealthCheck(message) - - -def perform_health_checks(random, settings, test_runner, search_strategy): - # Tell pytest to omit the body of this function from tracebacks - __tracebackhide__ = True - if not settings.perform_health_check: - return - if not Settings.default.perform_health_check: - return - - health_check_random = Random(random.getrandbits(128)) - # We "pre warm" the health check with one draw to give it some - # time to calculate any cached data. This prevents the case - # where the first draw of the health check takes ages because - # of loading unicode data the first time. - data = ConjectureData( - max_length=settings.buffer_size, - draw_bytes=lambda data, n: uniform(health_check_random, n) - ) - with Settings(settings, verbosity=Verbosity.quiet): - try: - test_runner(data, reify_and_execute( - search_strategy, - lambda *args, **kwargs: None, - )) - except BaseException: - escalate_hypothesis_internal_error() - count = 0 - overruns = 0 - filtered_draws = 0 - start = time.time() - while ( - count < 10 and time.time() < start + 1 and - filtered_draws < 50 and overruns < 20 - ): - try: - data = ConjectureData( - max_length=settings.buffer_size, - draw_bytes=lambda data, n: uniform(health_check_random, n) - ) - with Settings(settings, verbosity=Verbosity.quiet): - test_runner(data, reify_and_execute( - search_strategy, - lambda *args, **kwargs: None, - )) - count += 1 - except UnsatisfiedAssumption: - filtered_draws += 1 - except StopTest: - if data.status == Status.INVALID: - filtered_draws += 1 - else: - assert data.status == Status.OVERRUN - overruns += 1 - except InvalidArgument: - raise - - if overruns >= 20 or ( - not count and overruns > 0 - ): - fail_health_check(settings, ( - 'Examples routinely exceeded the max allowable size. ' - '(%d examples overran while generating %d valid ones)' - '. Generating examples this large will usually lead to' - ' bad results. You should try setting average_size or ' - 'max_size parameters on your collections and turning ' - 'max_leaves down on recursive() calls.') % ( - overruns, count - ), HealthCheck.data_too_large) - if filtered_draws >= 50 or ( - not count and filtered_draws > 0 - ): - fail_health_check(settings, ( - 'It looks like your strategy is filtering out a lot ' - 'of data. Health check found %d filtered examples but ' - 'only %d good ones. This will make your tests much ' - 'slower, and also will probably distort the data ' - 'generation quite a lot. You should adapt your ' - 'strategy to filter less. This can also be caused by ' - 'a low max_leaves parameter in recursive() calls') % ( - filtered_draws, count - ), HealthCheck.filter_too_much) - runtime = time.time() - start - if runtime > 1.0 or count < 10: - fail_health_check(settings, ( - 'Data generation is extremely slow: Only produced ' - '%d valid examples in %.2f seconds (%d invalid ones ' - 'and %d exceeded maximum size). Try decreasing ' - "size of the data you're generating (with e.g." - 'average_size or max_leaves parameters).' - ) % (count, runtime, filtered_draws, overruns), - HealthCheck.too_slow, - ) - - def get_random_for_wrapped_test(test, wrapped_test): settings = wrapped_test._hypothesis_internal_use_settings wrapped_test._hypothesis_internal_use_generated_seed = None @@ -496,9 +384,6 @@ args=new_args, kwonlyargs=new_kwonlyargs, annotations=annots) -HUNG_TEST_TIME_LIMIT = 5 * 60 - - ROOT = os.path.dirname(__file__) STDLIB = os.path.dirname(os.__file__) @@ -668,15 +553,6 @@ collector.save_data = save_data def evaluate_test_data(self, data): - if ( - time.time() - self.start_time >= HUNG_TEST_TIME_LIMIT - ): - fail_health_check(self.settings, ( - 'Your test has been running for at least five minutes. This ' - 'is probably not what you intended, so by default Hypothesis ' - 'turns it into an error.' - ), HealthCheck.hung_test) - try: if self.collector is None: result = self.test_runner(data, reify_and_execute( @@ -986,9 +862,6 @@ return try: - perform_health_checks( - random, settings, test_runner, search_strategy) - state = StateForActualGivenExecution( test_runner, search_strategy, test, settings, random) state.run() @@ -1030,6 +903,7 @@ min_satisfying_examples=0, max_shrinks=2000, ) + settings = Settings(settings, perform_health_check=False) if database_key is None and settings.database is not None: database_key = function_digest(condition) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/internal/compat.py new/hypothesis-3.38.0/src/hypothesis/internal/compat.py --- old/hypothesis-3.37.0/src/hypothesis/internal/compat.py 2017-11-12 21:10:23.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/internal/compat.py 2017-11-18 11:26:22.000000000 +0100 @@ -23,6 +23,7 @@ import re import sys import math +import time import codecs import platform import importlib @@ -114,7 +115,8 @@ struct_pack = struct.pack struct_unpack = struct.unpack - from time import monotonic as benchmark_time + def benchmark_time(): + return time.monotonic() else: import struct @@ -240,7 +242,8 @@ def quiet_raise(exc): raise exc - from time import time as benchmark_time + def benchmark_time(): + return time.time() # coverage mixes unicode and str filepaths on Python 2, which causes us diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/internal/conjecture/data.py new/hypothesis-3.38.0/src/hypothesis/internal/conjecture/data.py --- old/hypothesis-3.37.0/src/hypothesis/internal/conjecture/data.py 2017-11-12 21:10:23.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/internal/conjecture/data.py 2017-11-18 11:26:22.000000000 +0100 @@ -75,6 +75,7 @@ self.capped_indices = {} self.interesting_origin = None self.tags = set() + self.draw_times = [] def __assert_not_frozen(self, name): if self.frozen: @@ -129,11 +130,14 @@ if not at_top_level: return strategy.do_draw(self) else: + start_time = benchmark_time() try: return strategy.do_draw(self) except BaseException as e: mark_for_escalation(e) raise + finally: + self.draw_times.append(benchmark_time() - start_time) finally: if not self.frozen: self.stop_example() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/internal/conjecture/engine.py new/hypothesis-3.38.0/src/hypothesis/internal/conjecture/engine.py --- old/hypothesis-3.37.0/src/hypothesis/internal/conjecture/engine.py 2017-11-12 21:10:23.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/internal/conjecture/engine.py 2017-11-18 11:26:22.000000000 +0100 @@ -17,24 +17,41 @@ from __future__ import division, print_function, absolute_import -import time import heapq from enum import Enum from random import Random, getrandbits from weakref import WeakKeyDictionary from collections import defaultdict +import attr + from hypothesis import settings as Settings -from hypothesis import Phase +from hypothesis import Phase, HealthCheck from hypothesis.reporting import debug_report from hypothesis.internal.compat import EMPTY_BYTES, Counter, ceil, \ - hbytes, hrange, int_to_text, int_to_bytes, bytes_from_list, \ - to_bytes_sequence, unicode_safe_repr + hbytes, hrange, int_to_text, int_to_bytes, benchmark_time, \ + bytes_from_list, to_bytes_sequence, unicode_safe_repr from hypothesis.utils.conventions import UniqueIdentifier +from hypothesis.internal.healthcheck import fail_health_check from hypothesis.internal.conjecture.data import MAX_DEPTH, Status, \ StopTest, ConjectureData from hypothesis.internal.conjecture.minimizer import minimize +# Tell pytest to omit the body of this module from tracebacks +# http://doc.pytest.org/en/latest/example/simple.html#writing-well-integrated-assertion-helpers +__tracebackhide__ = True + + +HUNG_TEST_TIME_LIMIT = 5 * 60 + + +@attr.s +class HealthCheckState(object): + valid_examples = attr.ib(default=0) + invalid_examples = attr.ib(default=0) + overrun_examples = attr.ib(default=0) + draw_times = attr.ib(default=attr.Factory(list)) + class ExitReason(Enum): max_examples = 0 @@ -62,7 +79,7 @@ self.call_count = 0 self.event_call_counts = Counter() self.valid_examples = 0 - self.start_time = time.time() + self.start_time = benchmark_time() self.random = random or Random(getrandbits(128)) self.database_key = database_key self.status_runtimes = {} @@ -113,10 +130,21 @@ self.tag_intern_table = {} + self.health_check_state = None + def __tree_is_exhausted(self): return 0 in self.dead def test_function(self, data): + if ( + benchmark_time() - self.start_time >= HUNG_TEST_TIME_LIMIT + ): + fail_health_check(self.settings, ( + 'Your test has been running for at least five minutes. This ' + 'is probably not what you intended, so by default Hypothesis ' + 'turns it into an error.' + ), HealthCheck.hung_test) + self.call_count += 1 try: self._test_function(data) @@ -238,7 +266,7 @@ self.last_data = data if ( self.settings.timeout > 0 and - time.time() >= self.start_time + self.settings.timeout + benchmark_time() >= self.start_time + self.settings.timeout ): self.exit_with(ExitReason.timeout) @@ -253,6 +281,74 @@ if self.__tree_is_exhausted(): self.exit_with(ExitReason.finished) + self.record_for_health_check(data) + + def record_for_health_check(self, data): + # Once we've actually found a bug, there's no point in trying to run + # health checks - they'll just mask the actually important information. + if data.status == Status.INTERESTING: + self.health_check_state = None + + state = self.health_check_state + + if state is None: + return + + state.draw_times.extend(data.draw_times) + + if data.status == Status.VALID: + state.valid_examples += 1 + elif data.status == Status.INVALID: + state.invalid_examples += 1 + else: + assert data.status == Status.OVERRUN + state.overrun_examples += 1 + + max_valid_draws = 10 + max_invalid_draws = 50 + max_overrun_draws = 20 + + assert state.valid_examples <= max_valid_draws + + if state.valid_examples == max_valid_draws: + self.health_check_state = None + return + + if state.overrun_examples == max_overrun_draws: + fail_health_check(self.settings, ( + 'Examples routinely exceeded the max allowable size. ' + '(%d examples overran while generating %d valid ones)' + '. Generating examples this large will usually lead to' + ' bad results. You could try setting max_size parameters ' + 'on your collections and turning ' + 'max_leaves down on recursive() calls.') % ( + state.overrun_examples, state.valid_examples + ), HealthCheck.data_too_large) + if state.invalid_examples == max_invalid_draws: + fail_health_check(self.settings, ( + 'It looks like your strategy is filtering out a lot ' + 'of data. Health check found %d filtered examples but ' + 'only %d good ones. This will make your tests much ' + 'slower, and also will probably distort the data ' + 'generation quite a lot. You should adapt your ' + 'strategy to filter less. This can also be caused by ' + 'a low max_leaves parameter in recursive() calls') % ( + state.invalid_examples, state.valid_examples + ), HealthCheck.filter_too_much) + + draw_time = sum(state.draw_times) + + if draw_time > 1.0: + fail_health_check(self.settings, ( + 'Data generation is extremely slow: Only produced ' + '%d valid examples in %.2f seconds (%d invalid ones ' + 'and %d exceeded maximum size). Try decreasing ' + "size of the data you're generating (with e.g." + 'max_size or max_leaves parameters).' + ) % ( + state.valid_examples, draw_time, state.invalid_examples, + state.overrun_examples), HealthCheck.too_slow,) + def save_buffer(self, buffer, key=None): if self.settings.database is not None: if key is None: @@ -650,6 +746,8 @@ draw_bytes=lambda data, n: self.__rewrite_for_novelty( data, hbytes(n))) self.test_function(zero_data) + if self.settings.perform_health_check: + self.health_check_state = HealthCheckState() count = 0 while count < 10 and not self.interesting_examples: @@ -748,7 +846,7 @@ def _run(self): self.last_data = None - self.start_time = time.time() + self.start_time = benchmark_time() self.reuse_existing_examples() self.generate_new_examples() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/internal/healthcheck.py new/hypothesis-3.38.0/src/hypothesis/internal/healthcheck.py --- old/hypothesis-3.37.0/src/hypothesis/internal/healthcheck.py 1970-01-01 01:00:00.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/internal/healthcheck.py 2017-11-18 11:26:22.000000000 +0100 @@ -0,0 +1,38 @@ +# coding=utf-8 +# +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis-python +# +# Most of this work is copyright (C) 2013-2017 David R. MacIver +# (da...@drmaciver.com), but it contains contributions by others. See +# CONTRIBUTING.rst for a full list of people who may hold copyright, and +# consult the git log if you need to determine who owns an individual +# contribution. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. +# +# END HEADER + +from __future__ import division, print_function, absolute_import + +from hypothesis.errors import FailedHealthCheck + + +def fail_health_check(settings, message, label): + # Tell pytest to omit the body of this function from tracebacks + # http://doc.pytest.org/en/latest/example/simple.html#writing-well-integrated-assertion-helpers + __tracebackhide__ = True + + if label in settings.suppress_health_check: + return + if not settings.perform_health_check: + return + message += ( + '\nSee https://hypothesis.readthedocs.io/en/latest/health' + 'checks.html for more information about this. ' + 'If you want to disable just this health check, add %s ' + 'to the suppress_health_check settings for this test.' + ) % (label,) + raise FailedHealthCheck(message) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis/version.py new/hypothesis-3.38.0/src/hypothesis/version.py --- old/hypothesis-3.37.0/src/hypothesis/version.py 2017-11-12 21:11:25.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis/version.py 2017-11-18 11:27:45.000000000 +0100 @@ -17,5 +17,5 @@ from __future__ import division, print_function, absolute_import -__version_info__ = (3, 37, 0) +__version_info__ = (3, 38, 0) __version__ = '.'.join(map(str, __version_info__)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis.egg-info/PKG-INFO new/hypothesis-3.38.0/src/hypothesis.egg-info/PKG-INFO --- old/hypothesis-3.37.0/src/hypothesis.egg-info/PKG-INFO 2017-11-12 21:11:25.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis.egg-info/PKG-INFO 2017-11-18 11:27:46.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: hypothesis -Version: 3.37.0 +Version: 3.38.0 Summary: A library for property based testing Home-page: https://github.com/HypothesisWorks/hypothesis-python Author: David R. MacIver diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/hypothesis-3.37.0/src/hypothesis.egg-info/SOURCES.txt new/hypothesis-3.38.0/src/hypothesis.egg-info/SOURCES.txt --- old/hypothesis-3.37.0/src/hypothesis.egg-info/SOURCES.txt 2017-11-12 21:11:25.000000000 +0100 +++ new/hypothesis-3.38.0/src/hypothesis.egg-info/SOURCES.txt 2017-11-18 11:27:46.000000000 +0100 @@ -40,6 +40,7 @@ src/hypothesis/internal/detection.py src/hypothesis/internal/escalation.py src/hypothesis/internal/floats.py +src/hypothesis/internal/healthcheck.py src/hypothesis/internal/intervalsets.py src/hypothesis/internal/lazyformat.py src/hypothesis/internal/reflection.py
