Patchwork testing: generate tests operations using Hypothesis

login
register
mail settings
Submitter David MacIver
Date Feb. 23, 2016, 5:42 p.m.
Message ID <a3e91526080d164c339e.1456249351@laser-shark>
Download mbox | patch
Permalink /patch/13320/
State Superseded
Headers show

Comments

David MacIver - Feb. 23, 2016, 5:42 p.m.
# HG changeset patch
# User David R. MacIver <david@drmaciver.com>
# Date 1456235047 0
#      Tue Feb 23 13:44:07 2016 +0000
# Node ID a3e91526080d164c339eb73ab0241f17f46881d1
# Parent  83fc0c0556644d742e3d6eecfb38e742005799cc
testing: generate tests operations using Hypothesis

The idea of this patch is to expand the use of Hypothesis
within Mercurial to use its concept of "stateful testing".

The result is a test which runs a sequence of operations
against a family of Mercurial repositories. Each operation
is given a set of allowed ways it can fail. Any other
non-zero exit code is a test failure.

At the end, the whole sequence is then reverified by
generating a .t test and testing it again in pure
mode (this is also useful for catching non-determinism
bugs).

This has proven reasonably effective at finding bugs,
and has identified two problems in the shelve extension
already (issue5113 and issue5112).
David MacIver - Feb. 23, 2016, 5:44 p.m.
I updated this to fix a few things:

* a bug in the original (failing to properly quote an argument in bash)
* the default profile had poor behaviour when loading a lot of now bad data
from the database
* Some spacing issues for style guid econformance.

On 23 February 2016 at 17:42, David R. MacIver <david@drmaciver.com> wrote:

> # HG changeset patch
> # User David R. MacIver <david@drmaciver.com>
> # Date 1456235047 0
> #      Tue Feb 23 13:44:07 2016 +0000
> # Node ID a3e91526080d164c339eb73ab0241f17f46881d1
> # Parent  83fc0c0556644d742e3d6eecfb38e742005799cc
> testing: generate tests operations using Hypothesis
>
> The idea of this patch is to expand the use of Hypothesis
> within Mercurial to use its concept of "stateful testing".
>
> The result is a test which runs a sequence of operations
> against a family of Mercurial repositories. Each operation
> is given a set of allowed ways it can fail. Any other
> non-zero exit code is a test failure.
>
> At the end, the whole sequence is then reverified by
> generating a .t test and testing it again in pure
> mode (this is also useful for catching non-determinism
> bugs).
>
> This has proven reasonably effective at finding bugs,
> and has identified two problems in the shelve extension
> already (issue5113 and issue5112).
>
> diff -r 83fc0c055664 -r a3e91526080d .hgignore
> --- a/.hgignore Tue Jan 19 18:20:13 2016 +0000
> +++ b/.hgignore Tue Feb 23 13:44:07 2016 +0000
> @@ -21,6 +21,7 @@
>  .\#*
>  tests/.coverage*
>  tests/.testtimes*
> +tests/.hypothesis
>  tests/annotated
>  tests/*.err
>  tests/htmlcov
> diff -r 83fc0c055664 -r a3e91526080d tests/test-verify-repo-operations.py
> --- /dev/null   Thu Jan 01 00:00:00 1970 +0000
> +++ b/tests/test-verify-repo-operations.py      Tue Feb 23 13:44:07 2016
> +0000
> @@ -0,0 +1,542 @@
> +from __future__ import print_function, absolute_import
> +
> +"""Fuzz testing for operations against a Mercurial repository
> +
> +This uses Hypothesis's stateful testing to generate random repository
> +operations and test Mercurial using them, both to see if there are any
> +unexpected errors and to compare different versions of it."""
> +
> +import base64
> +from contextlib import contextmanager
> +import errno
> +import os
> +import pipes
> +import shutil
> +import silenttestrunner
> +import subprocess
> +
> +from hypothesis.extra.datetime import datetimes
> +from hypothesis.errors import HypothesisException
> +from hypothesis.stateful import (
> +    rule, RuleBasedStateMachine, Bundle, precondition)
> +import hypothesis.strategies as st
> +from hypothesis import settings, note
> +from hypothesis.configuration import set_hypothesis_home_dir
> +from hypothesis.database import ExampleDatabase
> +
> +testdir = os.path.abspath(os.environ["TESTDIR"])
> +
> +# We store Hypothesis examples here rather in the temporary test directory
> +# so that when rerunning a failing test this always results in refinding
> the
> +# previous failure. This directory is in .hgignore and should not be
> checked in
> +# but is useful to have for development.
> +set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis"))
> +
> +runtests = os.path.join(testdir, "run-tests.py")
> +testtmp = os.environ["TESTTMP"]
> +assert os.path.isdir(testtmp)
> +
> +generatedtests = os.path.join(testdir, "hypothesis-generated")
> +
> +try:
> +    os.makedirs(generatedtests)
> +except OSError:
> +    pass
> +
> +# We write out generated .t files to a file in order to ease debugging
> and to
> +# give a starting point for turning failures Hypothesis finds into normal
> +# tests. In order to ensure that multiple copies of this test can be run
> in
> +# parallel we use atomic file create to ensure that we always get a unique
> +# name.
> +file_index = 0
> +while True:
> +    file_index += 1
> +    savefile = os.path.join(generatedtests, "test-generated-%d.t" % (
> +        file_index,
> +    ))
> +    try:
> +        os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
> +        break
> +    except OSError as e:
> +        if e.errno != errno.EEXIST:
> +            raise
> +assert os.path.exists(savefile)
> +
> +hgrc = os.path.join(".hg", "hgrc")
> +
> +pathcharacters = (
> +    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
> +    "[]^_`;=@{}~ !#$%&'()+,-"
> +)
> +
> +files = st.text(pathcharacters, min_size=1).map(lambda x:
> x.strip()).filter(
> +    bool).map(lambda s: s.encode('ascii'))
> +
> +safetext = st.text(st.characters(
> +    min_codepoint=1, max_codepoint=127,
> +    blacklist_categories=('Cc', 'Cs')), min_size=1).map(
> +    lambda s: s.encode('utf-8')
> +)
> +
> +@contextmanager
> +def acceptableerrors(*args):
> +    """Sometimes we know an operation we're about to perform might fail,
> and
> +    we're OK with some of the failures. In those cases this may be used
> as a
> +    context manager and will swallow expected failures, as identified by
> +    substrings of the error message Mercurial emits."""
> +    try:
> +        yield
> +    except subprocess.CalledProcessError as e:
> +        if not any(a in e.output for a in args):
> +            note(e.output)
> +            raise
> +
> +reponames = st.text("abcdefghijklmnopqrstuvwxyz01234556789",
> min_size=1).map(
> +    lambda s: s.encode('ascii')
> +)
> +
> +class verifyingstatemachine(RuleBasedStateMachine):
> +    """This defines the set of acceptable operations on a Mercurial
> repository
> +    using Hypothesis's RuleBasedStateMachine.
> +
> +    The general concept is that we manage multiple repositories inside a
> +    repos/ directory in our temporary test location. Some of these are
> freshly
> +    inited, some are clones of the others. Our current working directory
> is
> +    always inside one of these repositories while the tests are running.
> +
> +    Hypothesis then performs a series of operations against these
> repositories,
> +    including hg commands, generating contents and editing the .hgrc file.
> +    If these operations fail in unexpected ways or behave differently in
> +    different configurations of Mercurial, the test will fail and a
> minimized
> +    .t test file will be written to the hypothesis-generated directory to
> +    exhibit that failure.
> +
> +    Operations are defined as methods with @rule() decorators. See the
> +    Hypothesis documentation at
> +    http://hypothesis.readthedocs.org/en/release/stateful.html for more
> +    details."""
> +
> +    # A bundle is a reusable collection of data that rules have previously
> +    # generated which may be provided as arguments to future rules.
> +    repos = Bundle('repos')
> +    paths = Bundle('paths')
> +    contents = Bundle('contents')
> +    branches = Bundle('branches')
> +    committimes = Bundle('committimes')
> +
> +    def __init__(self):
> +        super(verifyingstatemachine, self).__init__()
> +        self.repodir = os.path.join(testtmp, "repos")
> +        assert self.repodir.startswith("/tmp/")
> +        if os.path.exists(self.repodir):
> +            shutil.rmtree(self.repodir)
> +        os.chdir(testtmp)
> +        self.log = []
> +        self.failed = False
> +
> +        self.mkdirp("repos")
> +        self.cd("repos")
> +        self.mkdirp("repo1")
> +        self.cd("repo1")
> +        self.hg("init")
> +        self.extensions = {}
> +
> +    def teardown(self):
> +        """On teardown we clean up after ourselves as usual, but we also
> +        do some additional testing: We generate a .t file based on our
> test
> +        run using run-test.py -i to get the correct output.
> +
> +        We then test it in a number of other configurations, verifying
> that
> +        each passes the same test."""
> +        super(verifyingstatemachine, self).teardown()
> +        try:
> +            shutil.rmtree(self.repodir)
> +        except OSError:
> +            pass
> +        ttest = "\n".join("  " + l for l in self.log)
> +        os.chdir(testtmp)
> +        path = os.path.join(testtmp, "test-generated.t")
> +        with open(path, 'w') as o:
> +            o.write(ttest)
> +            o.write('\n')
> +            o.close()
> +        with open("/dev/null", "w") as devnull:
> +            rewriter = subprocess.Popen(
> +                [runtests, "--local", "-i", path], stdin=subprocess.PIPE,
> +                stdout=devnull, stderr=devnull,
> +            )
> +            rewriter.communicate("yes")
> +            with open(path, 'r') as i:
> +                ttest = i.read()
> +
> +        e = None
> +        if not self.failed:
> +            try:
> +                output = subprocess.check_output([
> +                    runtests, path, "--local", "--pure"
> +                ], stderr=subprocess.STDOUT)
> +                assert "Ran 1 test" in output, output
> +            except subprocess.CalledProcessError as e:
> +                note(e.output)
> +            finally:
> +                os.unlink(path)
> +                try:
> +                    os.unlink(path + ".err")
> +                except OSError:
> +                    pass
> +        if self.failed or e is not None:
> +            with open(savefile, "wb") as o:
> +                o.write(ttest)
> +        if e is not None:
> +            raise e
> +
> +    @property
> +    def currentrepo(self):
> +        return os.path.basename(os.getcwd())
> +
> +    def mkdirp(self, path):
> +        if os.path.exists(path):
> +            return
> +        self.log.append(
> +            "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),))
> +        os.makedirs(path)
> +
> +    def cd(self, path):
> +        path = os.path.relpath(path)
> +        if path == ".":
> +            return
> +        os.chdir(path)
> +        self.log.append("$ cd -- %s" % (pipes.quote(path),))
> +
> +    def hasextension(self, extension):
> +        repo = self.currentrepo
> +        return repo in self.extensions and extension in
> self.extensions[repo]
> +
> +    @rule(extension=st.sampled_from((
> +        'shelve', 'mq'
> +    )))
> +    def addextension(self, extension):
> +        extensions = self.extensions.setdefault(self.currentrepo, set())
> +        if extension in extensions:
> +            return
> +        extensions.add(extension)
> +        if not os.path.exists(hgrc):
> +            self.command("touch", hgrc)
> +        with open(hgrc, "r") as i:
> +            existing = i.read()
> +        if "[extensions]" not in existing:
> +            with open(hgrc, "a") as o:
> +                o.write("[extensions]\n")
> +            self.log.append("$ echo '[extensions]' >> %s" % (hgrc,))
> +        with open(hgrc, 'a') as o:
> +            line = "%s=" % (extension,)
> +            o.write(line)
> +            o.write("\n")
> +            self.log.append("$ echo %s >> %s" % (line, hgrc))
> +
> +    @rule()
> +    def dumprc(self):
> +        if os.path.exists(hgrc):
> +            self.command("cat", hgrc)
> +
> +    def hg(self, *args):
> +        self.command("hg", *args)
> +
> +    def command(self, *args):
> +        self.log.append("$ " + ' '.join(map(pipes.quote, args)))
> +        subprocess.check_output(args, stderr=subprocess.STDOUT)
> +
> +    def execute_step(self, step):
> +        try:
> +            return super(verifyingstatemachine, self).execute_step(step)
> +        except (HypothesisException, KeyboardInterrupt):
> +            raise
> +        except Exception:
> +            self.failed = True
> +            raise
> +
> +    @rule(
> +        target=paths,
> +        source=st.lists(files, min_size=1).map(
> +            lambda l: os.path.join(*l)).filter(
> +                lambda x: x and x[0] != "/" and x[-1] != "/"))
> +    def genpath(self, source):
> +        return source
> +
> +    @rule(
> +        target=committimes,
> +        when=datetimes(min_year=1970, max_year=2038) | st.none())
> +    def gentime(self, when):
> +        return when
> +
> +    @rule(
> +        target=repos,
> +        source=repos,
> +        name=reponames,
> +    )
> +    def clone(self, source, name):
> +        if not os.path.exists(os.path.join("..", name)):
> +            self.cd("..")
> +            self.hg("clone", source, name)
> +            self.cd(name)
> +        return name
> +
> +    @rule(
> +        target=repos,
> +        name=reponames,
> +    )
> +    def fresh(self, name):
> +        if not os.path.exists(os.path.join("..", name)):
> +            self.cd("..")
> +            self.mkdirp(name)
> +            self.cd(name)
> +            self.hg("init")
> +        return name
> +
> +    @rule(name=repos)
> +    def switch(self, name):
> +        self.cd(os.path.join("..", name))
> +        assert self.currentrepo == name
> +        assert os.path.exists(".hg")
> +
> +    @rule(target=repos)
> +    def origin(self):
> +        return "repo1"
> +
> +    @rule()
> +    def pull(self, repo=repos):
> +        with acceptableerrors("repository default not found"):
> +            self.hg("pull")
> +
> +    @rule(newbranch=st.booleans())
> +    def push(self, newbranch):
> +        with acceptableerrors(
> +            "default repository not configured",
> +            "no changes found",
> +        ):
> +            if newbranch:
> +                self.hg("push", "--new-branch")
> +            else:
> +                with acceptableerrors(
> +                    "creates new branches"
> +                ):
> +                    self.hg("push")
> +
> +    @rule(
> +        target=contents,
> +        content=st.one_of(
> +            st.binary(),
> +            st.text().map(lambda x: x.encode('utf-8'))
> +        ))
> +    def gencontent(self, content):
> +        return content
> +
> +    @rule(
> +        target=branches,
> +        name=safetext,
> +    )
> +    def genbranch(self, name):
> +        return name
> +
> +    @rule(target=paths, source=paths)
> +    def lowerpath(self, source):
> +        return source.lower()
> +
> +    @rule(target=paths, source=paths)
> +    def upperpath(self, source):
> +        return source.upper()
> +
> +    @rule()
> +    def log(self):
> +        self.hg("log")
> +
> +    @rule()
> +    def verify(self):
> +        self.hg("verify")
> +
> +    @rule()
> +    def diff(self):
> +        self.hg("diff", "--nodates")
> +
> +    @rule()
> +    def status(self):
> +        self.hg("status")
> +
> +    @rule()
> +    def checkbranch(self):
> +        self.hg("branch")
> +
> +    @rule(branch=branches)
> +    def switchbranch(self, branch):
> +        with acceptableerrors(
> +            'cannot use an integer as a name',
> +            'be used in a name',
> +            'a branch of the same name already exists',
> +            'is reserved',
> +        ):
> +            self.hg("branch", "--", branch)
> +
> +    @rule(branch=branches, c=st.booleans())
> +    def update(self, branch, c):
> +        with acceptableerrors(
> +            'unknown revision',
> +            'parse error',
> +        ):
> +            if c:
> +                self.hg("update", "-C", "--", branch)
> +            else:
> +                self.hg("update", "--", branch)
> +
> +    @rule(s=st.none() | st.integers(0, 100))
> +    def addremove(self, s):
> +        args = ["addremove"]
> +        if s is not None:
> +            args.extend(["-s", str(s)])
> +        self.hg(*args)
> +
> +    @rule(path=paths)
> +    def addpath(self, path):
> +        if os.path.exists(path):
> +            self.hg("add", "--", path)
> +
> +    @rule(path=paths)
> +    def forgetpath(self, path):
> +        if os.path.exists(path):
> +            with acceptableerrors("file is already untracked"):
> +                self.hg("forget", "--", path)
> +
> +    @rule(path=paths)
> +    def removepath(self, path):
> +        if os.path.exists(path):
> +            with acceptableerrors(
> +                'file is untracked', 'file has been marked for add',
> +                'file is modified',
> +            ):
> +                self.hg("remove", "--", path)
> +
> +    @rule(
> +        message=safetext,
> +        amend=st.booleans(),
> +        addremove=st.booleans(),
> +        secret=st.booleans(),
> +        close_branch=st.booleans(),
> +        when=committimes,
> +    )
> +    def maybecommit(
> +        self, message, amend, when, addremove, secret, close_branch
> +    ):
> +        command = ["commit"]
> +        errors = ["nothing changed"]
> +        if amend:
> +            errors.append("cannot amend public changesets")
> +            command.append("--amend")
> +        command.append("-m" + pipes.quote(message))
> +        if secret:
> +            command.append("--secret")
> +        if close_branch:
> +            command.append("--close-branch")
> +            errors.append("can only close branch heads")
> +        if addremove:
> +            command.append("--addremove")
> +        if when is not None:
> +            if when.year == 1970:
> +                errors.append('negative date value')
> +            if when.year == 2038:
> +                errors.append('exceeds 32 bits')
> +            command.append("--date=%s" % (
> +                when.strftime('%Y-%m-%d %H:%M:%S %z'),))
> +
> +        with acceptableerrors(*errors):
> +            self.hg(*command)
> +
> +    @rule(path=paths, content=contents)
> +    def writecontent(self, path, content):
> +        self.unadded_changes = True
> +        if os.path.isdir(path):
> +            return
> +        parent = os.path.dirname(path)
> +        if parent:
> +            self.mkdirp(parent)
> +        with open(path, 'wb') as o:
> +            o.write(content)
> +        self.log.append("$ echo %s | base64 --decode > %s" % (
> +            pipes.quote(base64.b64encode(content)),
> +            pipes.quote(path),
> +        ))
> +
> +    @rule()
> +    def export(self):
> +        self.hg("export")
> +
> +    @rule()
> +    @precondition(lambda self: self.hasextension("shelve"))
> +    def shelve(self):
> +        with acceptableerrors("nothing changed"):
> +            self.hg("shelve")
> +
> +    @rule()
> +    @precondition(lambda self: self.hasextension("shelve"))
> +    def unshelve(self):
> +        with acceptableerrors("no shelved changes to apply"):
> +            self.hg("unshelve")
> +
> +class writeonlydatabase(ExampleDatabase):
> +    def __init__(self, underlying):
> +        super(ExampleDatabase, self).__init__()
> +        self.underlying = underlying
> +
> +    def fetch(self, key):
> +        return ()
> +
> +    def save(self, key, value):
> +        self.underlying.save(key, value)
> +
> +    def delete(self, key, value):
> +        self.underlying.delete(key, value)
> +
> +    def close(self):
> +        self.underlying.close()
> +
> +settings.register_profile(
> +    'default',  settings(
> +        timeout=300,
> +        stateful_step_count=100,
> +        max_examples=20,
> +    )
> +)
> +
> +settings.register_profile(
> +    'fast',  settings(
> +        timeout=10,
> +        stateful_step_count=20,
> +        max_examples=5,
> +        min_satisfying_examples=1,
> +        max_shrinks=0,
> +    )
> +)
> +
> +settings.register_profile(
> +    'continuous', settings(
> +        timeout=-1,
> +        stateful_step_count=10000,
> +        max_examples=10 ** 8,
> +        max_iterations=10 ** 8,
> +        database=writeonlydatabase(settings.default.database)
> +    )
> +)
> +
> +settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default'))
> +
> +verifyingtest = verifyingstatemachine.TestCase
> +
> +verifyingtest.settings = settings.default
> +
> +if __name__ == '__main__':
> +    try:
> +        silenttestrunner.main(__name__)
> +    finally:
> +        # So as to prevent proliferation of useless test files, if we
> never
> +        # actually wrote a failing test we clean up after ourselves and
> delete
> +        # the file for doing so that we owned.
> +        if os.path.exists(savefile) and os.path.getsize(savefile) == 0:
> +            os.unlink(savefile)
>
Jun Wu - Feb. 23, 2016, 5:54 p.m.
It's better to use `hg email --flag V2` for updates so people can tell
it's an update from email title. Further updates will be V3, V4, etc.

On 02/23/2016 05:42 PM, David R. MacIver wrote:
> testing: generate tests operations using Hypothesis
David MacIver - Feb. 23, 2016, 5:57 p.m.
Thanks. Simon also pointed that out to me offline. Apologies for newbie
mistake. Will correct in future.

On 23 February 2016 at 17:54, Jun Wu <quark@fb.com> wrote:

> It's better to use `hg email --flag V2` for updates so people can tell
> it's an update from email title. Further updates will be V3, V4, etc.
>
> On 02/23/2016 05:42 PM, David R. MacIver wrote:
>
>> testing: generate tests operations using Hypothesis
>>
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
>
Martijn Pieters - Feb. 23, 2016, 5:58 p.m.
On 23 February 2016 at 18:44, David MacIver <david@drmaciver.com> wrote:
> I updated this to fix a few things:
>
> * a bug in the original (failing to properly quote an argument in bash)
> * the default profile had poor behaviour when loading a lot of now bad data
> from the database
> * Some spacing issues for style guid econformance.

When updating a patch, you can give it a version with the `--flag`
command-line option to `hg email`. `--flag V2` for a first update,
etc.

Patch

diff -r 83fc0c055664 -r a3e91526080d .hgignore
--- a/.hgignore	Tue Jan 19 18:20:13 2016 +0000
+++ b/.hgignore	Tue Feb 23 13:44:07 2016 +0000
@@ -21,6 +21,7 @@ 
 .\#*
 tests/.coverage*
 tests/.testtimes*
+tests/.hypothesis
 tests/annotated
 tests/*.err
 tests/htmlcov
diff -r 83fc0c055664 -r a3e91526080d tests/test-verify-repo-operations.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-verify-repo-operations.py	Tue Feb 23 13:44:07 2016 +0000
@@ -0,0 +1,542 @@ 
+from __future__ import print_function, absolute_import
+
+"""Fuzz testing for operations against a Mercurial repository
+
+This uses Hypothesis's stateful testing to generate random repository
+operations and test Mercurial using them, both to see if there are any
+unexpected errors and to compare different versions of it."""
+
+import base64
+from contextlib import contextmanager
+import errno
+import os
+import pipes
+import shutil
+import silenttestrunner
+import subprocess
+
+from hypothesis.extra.datetime import datetimes
+from hypothesis.errors import HypothesisException
+from hypothesis.stateful import (
+    rule, RuleBasedStateMachine, Bundle, precondition)
+import hypothesis.strategies as st
+from hypothesis import settings, note
+from hypothesis.configuration import set_hypothesis_home_dir
+from hypothesis.database import ExampleDatabase
+
+testdir = os.path.abspath(os.environ["TESTDIR"])
+
+# We store Hypothesis examples here rather in the temporary test directory
+# so that when rerunning a failing test this always results in refinding the
+# previous failure. This directory is in .hgignore and should not be checked in
+# but is useful to have for development.
+set_hypothesis_home_dir(os.path.join(testdir, ".hypothesis"))
+
+runtests = os.path.join(testdir, "run-tests.py")
+testtmp = os.environ["TESTTMP"]
+assert os.path.isdir(testtmp)
+
+generatedtests = os.path.join(testdir, "hypothesis-generated")
+
+try:
+    os.makedirs(generatedtests)
+except OSError:
+    pass
+
+# We write out generated .t files to a file in order to ease debugging and to
+# give a starting point for turning failures Hypothesis finds into normal
+# tests. In order to ensure that multiple copies of this test can be run in
+# parallel we use atomic file create to ensure that we always get a unique
+# name.
+file_index = 0
+while True:
+    file_index += 1
+    savefile = os.path.join(generatedtests, "test-generated-%d.t" % (
+        file_index,
+    ))
+    try:
+        os.close(os.open(savefile, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
+        break
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+assert os.path.exists(savefile)
+
+hgrc = os.path.join(".hg", "hgrc")
+
+pathcharacters = (
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+    "[]^_`;=@{}~ !#$%&'()+,-"
+)
+
+files = st.text(pathcharacters, min_size=1).map(lambda x: x.strip()).filter(
+    bool).map(lambda s: s.encode('ascii'))
+
+safetext = st.text(st.characters(
+    min_codepoint=1, max_codepoint=127,
+    blacklist_categories=('Cc', 'Cs')), min_size=1).map(
+    lambda s: s.encode('utf-8')
+)
+
+@contextmanager
+def acceptableerrors(*args):
+    """Sometimes we know an operation we're about to perform might fail, and
+    we're OK with some of the failures. In those cases this may be used as a
+    context manager and will swallow expected failures, as identified by
+    substrings of the error message Mercurial emits."""
+    try:
+        yield
+    except subprocess.CalledProcessError as e:
+        if not any(a in e.output for a in args):
+            note(e.output)
+            raise
+
+reponames = st.text("abcdefghijklmnopqrstuvwxyz01234556789", min_size=1).map(
+    lambda s: s.encode('ascii')
+)
+
+class verifyingstatemachine(RuleBasedStateMachine):
+    """This defines the set of acceptable operations on a Mercurial repository
+    using Hypothesis's RuleBasedStateMachine.
+
+    The general concept is that we manage multiple repositories inside a
+    repos/ directory in our temporary test location. Some of these are freshly
+    inited, some are clones of the others. Our current working directory is
+    always inside one of these repositories while the tests are running.
+
+    Hypothesis then performs a series of operations against these repositories,
+    including hg commands, generating contents and editing the .hgrc file.
+    If these operations fail in unexpected ways or behave differently in
+    different configurations of Mercurial, the test will fail and a minimized
+    .t test file will be written to the hypothesis-generated directory to
+    exhibit that failure.
+
+    Operations are defined as methods with @rule() decorators. See the
+    Hypothesis documentation at
+    http://hypothesis.readthedocs.org/en/release/stateful.html for more
+    details."""
+
+    # A bundle is a reusable collection of data that rules have previously
+    # generated which may be provided as arguments to future rules.
+    repos = Bundle('repos')
+    paths = Bundle('paths')
+    contents = Bundle('contents')
+    branches = Bundle('branches')
+    committimes = Bundle('committimes')
+
+    def __init__(self):
+        super(verifyingstatemachine, self).__init__()
+        self.repodir = os.path.join(testtmp, "repos")
+        assert self.repodir.startswith("/tmp/")
+        if os.path.exists(self.repodir):
+            shutil.rmtree(self.repodir)
+        os.chdir(testtmp)
+        self.log = []
+        self.failed = False
+
+        self.mkdirp("repos")
+        self.cd("repos")
+        self.mkdirp("repo1")
+        self.cd("repo1")
+        self.hg("init")
+        self.extensions = {}
+
+    def teardown(self):
+        """On teardown we clean up after ourselves as usual, but we also
+        do some additional testing: We generate a .t file based on our test
+        run using run-test.py -i to get the correct output.
+
+        We then test it in a number of other configurations, verifying that
+        each passes the same test."""
+        super(verifyingstatemachine, self).teardown()
+        try:
+            shutil.rmtree(self.repodir)
+        except OSError:
+            pass
+        ttest = "\n".join("  " + l for l in self.log)
+        os.chdir(testtmp)
+        path = os.path.join(testtmp, "test-generated.t")
+        with open(path, 'w') as o:
+            o.write(ttest)
+            o.write('\n')
+            o.close()
+        with open("/dev/null", "w") as devnull:
+            rewriter = subprocess.Popen(
+                [runtests, "--local", "-i", path], stdin=subprocess.PIPE,
+                stdout=devnull, stderr=devnull,
+            )
+            rewriter.communicate("yes")
+            with open(path, 'r') as i:
+                ttest = i.read()
+
+        e = None
+        if not self.failed:
+            try:
+                output = subprocess.check_output([
+                    runtests, path, "--local", "--pure"
+                ], stderr=subprocess.STDOUT)
+                assert "Ran 1 test" in output, output
+            except subprocess.CalledProcessError as e:
+                note(e.output)
+            finally:
+                os.unlink(path)
+                try:
+                    os.unlink(path + ".err")
+                except OSError:
+                    pass
+        if self.failed or e is not None:
+            with open(savefile, "wb") as o:
+                o.write(ttest)
+        if e is not None:
+            raise e
+
+    @property
+    def currentrepo(self):
+        return os.path.basename(os.getcwd())
+
+    def mkdirp(self, path):
+        if os.path.exists(path):
+            return
+        self.log.append(
+            "$ mkdir -p -- %s" % (pipes.quote(os.path.relpath(path)),))
+        os.makedirs(path)
+
+    def cd(self, path):
+        path = os.path.relpath(path)
+        if path == ".":
+            return
+        os.chdir(path)
+        self.log.append("$ cd -- %s" % (pipes.quote(path),))
+
+    def hasextension(self, extension):
+        repo = self.currentrepo
+        return repo in self.extensions and extension in self.extensions[repo]
+
+    @rule(extension=st.sampled_from((
+        'shelve', 'mq'
+    )))
+    def addextension(self, extension):
+        extensions = self.extensions.setdefault(self.currentrepo, set())
+        if extension in extensions:
+            return
+        extensions.add(extension)
+        if not os.path.exists(hgrc):
+            self.command("touch", hgrc)
+        with open(hgrc, "r") as i:
+            existing = i.read()
+        if "[extensions]" not in existing:
+            with open(hgrc, "a") as o:
+                o.write("[extensions]\n")
+            self.log.append("$ echo '[extensions]' >> %s" % (hgrc,))
+        with open(hgrc, 'a') as o:
+            line = "%s=" % (extension,)
+            o.write(line)
+            o.write("\n")
+            self.log.append("$ echo %s >> %s" % (line, hgrc))
+
+    @rule()
+    def dumprc(self):
+        if os.path.exists(hgrc):
+            self.command("cat", hgrc)
+
+    def hg(self, *args):
+        self.command("hg", *args)
+
+    def command(self, *args):
+        self.log.append("$ " + ' '.join(map(pipes.quote, args)))
+        subprocess.check_output(args, stderr=subprocess.STDOUT)
+
+    def execute_step(self, step):
+        try:
+            return super(verifyingstatemachine, self).execute_step(step)
+        except (HypothesisException, KeyboardInterrupt):
+            raise
+        except Exception:
+            self.failed = True
+            raise
+
+    @rule(
+        target=paths,
+        source=st.lists(files, min_size=1).map(
+            lambda l: os.path.join(*l)).filter(
+                lambda x: x and x[0] != "/" and x[-1] != "/"))
+    def genpath(self, source):
+        return source
+
+    @rule(
+        target=committimes,
+        when=datetimes(min_year=1970, max_year=2038) | st.none())
+    def gentime(self, when):
+        return when
+
+    @rule(
+        target=repos,
+        source=repos,
+        name=reponames,
+    )
+    def clone(self, source, name):
+        if not os.path.exists(os.path.join("..", name)):
+            self.cd("..")
+            self.hg("clone", source, name)
+            self.cd(name)
+        return name
+
+    @rule(
+        target=repos,
+        name=reponames,
+    )
+    def fresh(self, name):
+        if not os.path.exists(os.path.join("..", name)):
+            self.cd("..")
+            self.mkdirp(name)
+            self.cd(name)
+            self.hg("init")
+        return name
+
+    @rule(name=repos)
+    def switch(self, name):
+        self.cd(os.path.join("..", name))
+        assert self.currentrepo == name
+        assert os.path.exists(".hg")
+
+    @rule(target=repos)
+    def origin(self):
+        return "repo1"
+
+    @rule()
+    def pull(self, repo=repos):
+        with acceptableerrors("repository default not found"):
+            self.hg("pull")
+
+    @rule(newbranch=st.booleans())
+    def push(self, newbranch):
+        with acceptableerrors(
+            "default repository not configured",
+            "no changes found",
+        ):
+            if newbranch:
+                self.hg("push", "--new-branch")
+            else:
+                with acceptableerrors(
+                    "creates new branches"
+                ):
+                    self.hg("push")
+
+    @rule(
+        target=contents,
+        content=st.one_of(
+            st.binary(),
+            st.text().map(lambda x: x.encode('utf-8'))
+        ))
+    def gencontent(self, content):
+        return content
+
+    @rule(
+        target=branches,
+        name=safetext,
+    )
+    def genbranch(self, name):
+        return name
+
+    @rule(target=paths, source=paths)
+    def lowerpath(self, source):
+        return source.lower()
+
+    @rule(target=paths, source=paths)
+    def upperpath(self, source):
+        return source.upper()
+
+    @rule()
+    def log(self):
+        self.hg("log")
+
+    @rule()
+    def verify(self):
+        self.hg("verify")
+
+    @rule()
+    def diff(self):
+        self.hg("diff", "--nodates")
+
+    @rule()
+    def status(self):
+        self.hg("status")
+
+    @rule()
+    def checkbranch(self):
+        self.hg("branch")
+
+    @rule(branch=branches)
+    def switchbranch(self, branch):
+        with acceptableerrors(
+            'cannot use an integer as a name',
+            'be used in a name',
+            'a branch of the same name already exists',
+            'is reserved',
+        ):
+            self.hg("branch", "--", branch)
+
+    @rule(branch=branches, c=st.booleans())
+    def update(self, branch, c):
+        with acceptableerrors(
+            'unknown revision',
+            'parse error',
+        ):
+            if c:
+                self.hg("update", "-C", "--", branch)
+            else:
+                self.hg("update", "--", branch)
+
+    @rule(s=st.none() | st.integers(0, 100))
+    def addremove(self, s):
+        args = ["addremove"]
+        if s is not None:
+            args.extend(["-s", str(s)])
+        self.hg(*args)
+
+    @rule(path=paths)
+    def addpath(self, path):
+        if os.path.exists(path):
+            self.hg("add", "--", path)
+
+    @rule(path=paths)
+    def forgetpath(self, path):
+        if os.path.exists(path):
+            with acceptableerrors("file is already untracked"):
+                self.hg("forget", "--", path)
+
+    @rule(path=paths)
+    def removepath(self, path):
+        if os.path.exists(path):
+            with acceptableerrors(
+                'file is untracked', 'file has been marked for add',
+                'file is modified',
+            ):
+                self.hg("remove", "--", path)
+
+    @rule(
+        message=safetext,
+        amend=st.booleans(),
+        addremove=st.booleans(),
+        secret=st.booleans(),
+        close_branch=st.booleans(),
+        when=committimes,
+    )
+    def maybecommit(
+        self, message, amend, when, addremove, secret, close_branch
+    ):
+        command = ["commit"]
+        errors = ["nothing changed"]
+        if amend:
+            errors.append("cannot amend public changesets")
+            command.append("--amend")
+        command.append("-m" + pipes.quote(message))
+        if secret:
+            command.append("--secret")
+        if close_branch:
+            command.append("--close-branch")
+            errors.append("can only close branch heads")
+        if addremove:
+            command.append("--addremove")
+        if when is not None:
+            if when.year == 1970:
+                errors.append('negative date value')
+            if when.year == 2038:
+                errors.append('exceeds 32 bits')
+            command.append("--date=%s" % (
+                when.strftime('%Y-%m-%d %H:%M:%S %z'),))
+
+        with acceptableerrors(*errors):
+            self.hg(*command)
+
+    @rule(path=paths, content=contents)
+    def writecontent(self, path, content):
+        self.unadded_changes = True
+        if os.path.isdir(path):
+            return
+        parent = os.path.dirname(path)
+        if parent:
+            self.mkdirp(parent)
+        with open(path, 'wb') as o:
+            o.write(content)
+        self.log.append("$ echo %s | base64 --decode > %s" % (
+            pipes.quote(base64.b64encode(content)),
+            pipes.quote(path),
+        ))
+
+    @rule()
+    def export(self):
+        self.hg("export")
+
+    @rule()
+    @precondition(lambda self: self.hasextension("shelve"))
+    def shelve(self):
+        with acceptableerrors("nothing changed"):
+            self.hg("shelve")
+
+    @rule()
+    @precondition(lambda self: self.hasextension("shelve"))
+    def unshelve(self):
+        with acceptableerrors("no shelved changes to apply"):
+            self.hg("unshelve")
+
+class writeonlydatabase(ExampleDatabase):
+    def __init__(self, underlying):
+        super(ExampleDatabase, self).__init__()
+        self.underlying = underlying
+
+    def fetch(self, key):
+        return ()
+
+    def save(self, key, value):
+        self.underlying.save(key, value)
+
+    def delete(self, key, value):
+        self.underlying.delete(key, value)
+
+    def close(self):
+        self.underlying.close()
+
+settings.register_profile(
+    'default',  settings(
+        timeout=300,
+        stateful_step_count=100,
+        max_examples=20,
+    )
+)
+
+settings.register_profile(
+    'fast',  settings(
+        timeout=10,
+        stateful_step_count=20,
+        max_examples=5,
+        min_satisfying_examples=1,
+        max_shrinks=0,
+    )
+)
+
+settings.register_profile(
+    'continuous', settings(
+        timeout=-1,
+        stateful_step_count=10000,
+        max_examples=10 ** 8,
+        max_iterations=10 ** 8,
+        database=writeonlydatabase(settings.default.database)
+    )
+)
+
+settings.load_profile(os.getenv('HYPOTHESIS_PROFILE', 'default'))
+
+verifyingtest = verifyingstatemachine.TestCase
+
+verifyingtest.settings = settings.default
+
+if __name__ == '__main__':
+    try:
+        silenttestrunner.main(__name__)
+    finally:
+        # So as to prevent proliferation of useless test files, if we never
+        # actually wrote a failing test we clean up after ourselves and delete
+        # the file for doing so that we owned.
+        if os.path.exists(savefile) and os.path.getsize(savefile) == 0:
+            os.unlink(savefile)