Patchwork [V2] perf: add asv benchmarks

login
register
mail settings
Submitter Philippe Pepiot
Date Nov. 3, 2016, 3:58 p.m.
Message ID <67e096cea548a37ba80d.1478188693@sh71.tls.logilab.fr>
Download mbox | patch
Permalink /patch/17321/
State Superseded
Headers show

Comments

Philippe Pepiot - Nov. 3, 2016, 3:58 p.m.
# HG changeset patch
# User Philippe Pepiot <philippe.pepiot@logilab.fr>
# Date 1475136994 -7200
#      Thu Sep 29 10:16:34 2016 +0200
# Node ID 67e096cea548a37ba80ddf04e62a1cc1d50e9c96
# Parent  b032a7b676c6637b2ac6f3ef29431013b15a08f9
perf: add asv benchmarks

Airspeed velocity (ASV) is a python framework for benchmarking Python packages
over their lifetime. The results are displayed in an interactive web frontend.

Add ASV benchmarks for mercurial that use contrib/perf.py extension that could
be run against multiple reference repositories.

The benchmark suite now includes revsets from contrib/base-revsets.txt with
variants, perftags, perfstatus, perfmanifest and perfheads.

Installation requires asv>=0.2, python-hglib and virtualenv

This is part of PerformanceTrackingSuitePlan
https://www.mercurial-scm.org/wiki/PerformanceTrackingSuitePlan
Philippe Pepiot - Nov. 3, 2016, 4:21 p.m.
On 11/03/2016 04:58 PM, Philippe Pepiot wrote:

> # HG changeset patch
> # User Philippe Pepiot <philippe.pepiot@logilab.fr>
> # Date 1475136994 -7200
> #      Thu Sep 29 10:16:34 2016 +0200
> # Node ID 67e096cea548a37ba80ddf04e62a1cc1d50e9c96
> # Parent  b032a7b676c6637b2ac6f3ef29431013b15a08f9
> perf: add asv benchmarks

Here is my own review

> Airspeed velocity (ASV) is a python framework for benchmarking Python packages
> over their lifetime. The results are displayed in an interactive web frontend.
>
> Add ASV benchmarks for mercurial that use contrib/perf.py extension that could
> be run against multiple reference repositories.
>
> The benchmark suite now includes revsets from contrib/base-revsets.txt with
> variants, perftags, perfstatus, perfmanifest and perfheads.
>
> Installation requires asv>=0.2, python-hglib and virtualenv
>
> This is part of PerformanceTrackingSuitePlan
> https://www.mercurial-scm.org/wiki/PerformanceTrackingSuitePlan
>
> diff --git a/.hgignore b/.hgignore
> --- a/.hgignore
> +++ b/.hgignore
> @@ -49,6 +49,7 @@ mercurial.egg-info
>   tags
>   cscope.*
>   .idea/*
> +.asv/*
>   i18n/hg.pot
>   locale/*/LC_MESSAGES/hg.mo
>   hgext/__index__.py
> diff --git a/contrib/asv.conf.json b/contrib/asv.conf.json
> new file mode 100644
> --- /dev/null
> +++ b/contrib/asv.conf.json
> @@ -0,0 +1,127 @@
> +{
> +    // The version of the config file format.  Do not change, unless
> +    // you know what you are doing.
> +    "version": 1,
> +
> +    // The name of the project being benchmarked
> +    "project": "mercurial",
> +
> +    // The project's homepage
> +    "project_url": "http://mercurial-scm.org/",
> +
> +    // The URL or local path of the source code repository for the
> +    // project being benchmarked
> +    "repo": "..",
> +
> +    // List of branches to benchmark. If not provided, defaults to "master"
> +    // (for git) or "default" (for mercurial).
> +    // "branches": ["master"], // for git
> +    // "branches": ["default"],    // for mercurial
> +    "branches": ["default", "stable"],
> +
> +    // The DVCS being used.  If not set, it will be automatically
> +    // determined from "repo" by looking at the protocol in the URL
> +    // (if remote), or by looking for special directories, such as
> +    // ".git" (if local).
> +    // "dvcs": "git",
> +
> +    // The tool to use to create environments.  May be "conda",
> +    // "virtualenv" or other value depending on the plugins in use.
> +    // If missing or the empty string, the tool will be automatically
> +    // determined by looking for tools on the PATH environment
> +    // variable.
> +    "environment_type": "virtualenv",
> +
> +    // the base URL to show a commit for the project.
> +    "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/",
> +
> +    // The Pythons you'd like to test against.  If not provided, defaults
> +    // to the current version of Python used to run `asv`.
> +    // "pythons": ["2.7", "3.3"],
> +
> +    // The matrix of dependencies to test.  Each key is the name of a
> +    // package (in PyPI) and the values are version numbers.  An empty
> +    // list or empty string indicates to just test against the default
> +    // (latest) version. null indicates that the package is to not be
> +    // installed. If the package to be tested is only available from
> +    // PyPi, and the 'environment_type' is conda, then you can preface
> +    // the package name by 'pip+', and the package will be installed via
> +    // pip (with all the conda available packages installed first,
> +    // followed by the pip installed packages).
> +    //
> +    // "matrix": {
> +    //     "numpy": ["1.6", "1.7"],
> +    //     "six": ["", null],        // test with and without six installed
> +    //     "pip+emcee": [""],   // emcee is only available for install with pip.
> +    // },
> +
> +    // Combinations of libraries/python versions can be excluded/included
> +    // from the set to test. Each entry is a dictionary containing additional
> +    // key-value pairs to include/exclude.
> +    //
> +    // An exclude entry excludes entries where all values match. The
> +    // values are regexps that should match the whole string.
> +    //
> +    // An include entry adds an environment. Only the packages listed
> +    // are installed. The 'python' key is required. The exclude rules
> +    // do not apply to includes.
> +    //
> +    // In addition to package names, the following keys are available:
> +    //
> +    // - python
> +    //     Python version, as in the *pythons* variable above.
> +    // - environment_type
> +    //     Environment type, as above.
> +    // - sys_platform
> +    //     Platform, as in sys.platform. Possible values for the common
> +    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
> +    //
> +    // "exclude": [
> +    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
> +    //     {"environment_type": "conda", "six": null}, // don't run without six on conda
> +    // ],
> +    //
> +    // "include": [
> +    //     // additional env for python2.7
> +    //     {"python": "2.7", "numpy": "1.8"},
> +    //     // additional env if run on windows+conda
> +    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
> +    // ],
> +
> +    // The directory (relative to the current directory) that benchmarks are
> +    // stored in.  If not provided, defaults to "benchmarks"
> +    "benchmark_dir": "benchmarks",
> +
> +    // The directory (relative to the current directory) to cache the Python
> +    // environments in.  If not provided, defaults to "env"
> +    "env_dir": "../.asv/env",
> +
> +    // The directory (relative to the current directory) that raw benchmark
> +    // results are stored in.  If not provided, defaults to "results".
> +    "results_dir": "../.asv/results",
> +
> +    // The directory (relative to the current directory) that the html tree
> +    // should be written to.  If not provided, defaults to "html".
> +    "html_dir": "../.asv/html",
> +
> +    // The number of characters to retain in the commit hashes.
> +    // "hash_length": 8,
> +
> +    // `asv` will cache wheels of the recent builds in each
> +    // environment, making them faster to install next time.  This is
> +    // number of builds to keep, per environment.
> +    // "wheel_cache_size": 0
> +
> +    // The commits after which the regression search in `asv publish`
> +    // should start looking for regressions. Dictionary whose keys are
> +    // regexps matching to benchmark names, and values corresponding to
> +    // the commit (exclusive) after which to start looking for
> +    // regressions.  The default is to start from the first commit
> +    // with results. If the commit is `null`, regression detection is
> +    // skipped for the matching benchmark.
> +    //
> +    // "regressions_first_commits": {
> +    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
> +    //    "another_benchmark": null,   // Skip regression detection altogether
> +    // }
> +}
> diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/__init__.py
> @@ -0,0 +1,106 @@
> +"""ASV (https://asv.readthedocs.io) benchmark suite
> +
> +Benchmark are parameterized against reference repositories found in the
> +directory pointed by the REPOS_DIR environment variable.
> +
> +Invocation example:
> +
> +    $ export REPOS_DIR=~/hgperf/repos
> +    # run suite on given revision
> +    $ asv --config contrib/asv.conf.json run REV
> +    # run suite on new changesets found in stable and default branch
> +    $ asv --config contrib/asv.conf.json run NEW
> +    # display a comparative result table of benchmark results between two given
> +    # revisions
> +    $ asv --config contrib/asv.conf.json compare REV1 REV2
> +    # compute regression detection and generate ASV static website
> +    $ asv --config contrib/asv.conf.json publish
> +    # serve the static website
> +    $ asv --config contrib/asv.conf.json preview
> +"""
> +
> +from __future__ import absolute_import
> +
> +import contextlib
> +import functools
> +import io
> +import os
> +import re
> +
> +from mercurial import (
> +    extensions,
> +    hg,
> +    ui as uimod,
> +)
> +
> +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),
> +                          os.path.pardir, os.path.pardir))
> +reposdir = os.environ['REPOS_DIR']
> +reposnames = [name for name in os.listdir(reposdir)
> +              if os.path.isdir(os.path.join(reposdir, name, ".hg"))]
> +outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
> +                       r'\d+.\d+ \(best of \d+\)'))
> +
> +@contextlib.contextmanager
> +def redirectuioutput(ui):
> +    old = ui.fout, ui.ferr
> +    new = io.BytesIO()
> +    try:
> +        ui.fout = ui.ferr = new
> +        yield new
> +    finally:
> +        ui.fout, ui.ferr = old
> +

I'd like to use the pushbuffer()/popbuffer() API here, but there is a 
ui.copy() in contrib/perf.py that doesn't copy buffers. Not sure about 
how this should be done.

> +def runperfcommand(reponame, command, *args, **kwargs):
> +    os.environ["HGRCPATH"] = ""
> +    ui = uimod.ui()
> +    repo = hg.repository(ui, os.path.join(reposdir, reponame))
> +    perfext = extensions.load(ui, 'perfext',
> +                              os.path.join(basedir, 'contrib', 'perf.py'))
> +    cmd = getattr(perfext, command)
> +    with redirectuioutput(ui) as buf:
> +        cmd(ui, repo, *args, **kwargs)
> +    buf.seek(0)
> +    output = buf.read()
> +    match = outputre.search(output)
> +    if not match:
> +        raise ValueError("Invalid output {0}".format(output))
> +    return float(match.group(1))
> +
> +def perfbench(repos=reposnames, name=None, params=None):
> +    """decorator to declare ASV benchmark based on contrib/perf.py extension
> +
> +    An ASV benchmark is a python function with the given attributes:
> +
> +    __name__: should start with track_, time_ or mem_ to be collected by ASV
> +    params and param_name: parameter matrix to display multiple graphs on the
> +    same page.
> +    pretty_name: If defined it's displayed in web-ui instead of __name__
> +    (useful for revsets)
> +    the module name is prepended to the benchmark name and displayed as
> +    "category" in webui.
> +
> +    Benchmarks are automatically parameterized with repositories found in the
> +    REPOS_DIR environment variable.
> +
> +    `params` is the param matrix in the form of a list of tuple
> +    (param_name, [value0, value1])
> +
> +    For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
> +    (a, c), (a, d), (b, c) and (b, d).
> +    """
> +    params = list(params or [])
> +    params.insert(0, ("repo", repos))
> +
> +    def decorator(func):
> +        @functools.wraps(func)
> +        def wrapped(repo, *args):
> +            def perf(command, *a, **kw):
> +                return runperfcommand(repo, command, *a, **kw)
> +            return func(perf, *args)
> +
> +        wrapped.params = [p[1] for p in params]
> +        wrapped.param_names = [p[0] for p in params]
> +        wrapped.pretty_name = name
> +        return wrapped
> +    return decorator
> diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/perf.py
> @@ -0,0 +1,19 @@
> +from __future__ import absolute_import
> +
> +from . import perfbench
> +
> +@perfbench()
> +def track_tags(perf):
> +    return perf("perftags")
> +
> +@perfbench()
> +def track_status(perf):
> +    return perf("perfstatus", unknown=False)
> +
> +@perfbench(params=[('rev', ['1000', '10000', 'tip'])])
> +def track_manifest(perf, rev):
> +    return perf("perfmanifest", rev)
> +
> +@perfbench()
> +def track_heads(perf):
> +    return perf("perfheads")

ASV benchmark are categorized by their module name (relative to 
benchmark_dir parameter in asv.conf.json) and should follow name 
convention (track_, time_, mem_ etc), that's why they are in a separate 
file.
Since we're generating benchmarks that are just wrappers contrib/perf.py 
benchmarks. I'd prefer to describe them in a single file and more in a 
declarative form. That's the purpose of 
https://github.com/spacetelescope/asv/pull/484

> diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/revset.py
> @@ -0,0 +1,45 @@
> +"""ASV revset benchmarks generated from contrib/base-revsets.txt
> +
> +Each revset benchmark is parameterized with variants (first, last, sort, ...)
> +"""
> +from __future__ import absolute_import
> +
> +import os
> +import string
> +import sys
> +
> +from . import basedir, perfbench
> +
> +def createrevsetbenchmark(baseset, variants=None):
> +    if variants is None:
> +        # Default variants
> +        variants = ["plain", "first", "last", "sort", "sort+first",
> +                    "sort+last"]
> +    fname = "track_" + "_".join("".join([
> +        c if c in string.digits + string.letters else " "
> +        for c in baseset
> +    ]).split())
> +
> +    def wrap(fname, baseset):
> +        @perfbench(name=baseset, params=[("variant", variants)])
> +        def f(perf, variant):
> +            revset = baseset
> +            if variant != "plain":
> +                for var in variant.split("+"):
> +                    revset = "%s(%s)" % (var, revset)
> +            return perf("perfrevset", revset)
> +        f.__name__ = fname
> +        return f
> +    return wrap(fname, baseset)
> +
> +def initializerevsetbenchmarks():
> +    mod = sys.modules[__name__]
> +    with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'),
> +              'rb') as fh:
> +        for line in fh:
> +            baseset = line.strip()
> +            if baseset and not baseset.startswith('#'):
> +                func = createrevsetbenchmark(baseset)
> +                setattr(mod, func.__name__, func)
> +
> +initializerevsetbenchmarks()
Jun Wu - Nov. 14, 2016, 10:02 a.m.
Generally looks good to me. Some inline comments about small issues.

I wonder if it is possible to get rid of the dependencies on virtualenv and
hglib as installing them could be troublesome in some environments. I think
"virtualenv" could be replaced by just copying the checked out files to
somewhere, and hglib seems redundant.

Excerpts from Philippe Pepiot's message of 2016-11-03 16:58:13 +0100:
> # HG changeset patch
> # User Philippe Pepiot <philippe.pepiot@logilab.fr>
> # Date 1475136994 -7200
> #      Thu Sep 29 10:16:34 2016 +0200
> # Node ID 67e096cea548a37ba80ddf04e62a1cc1d50e9c96
> # Parent  b032a7b676c6637b2ac6f3ef29431013b15a08f9
> perf: add asv benchmarks
> 
> Airspeed velocity (ASV) is a python framework for benchmarking Python packages
> over their lifetime. The results are displayed in an interactive web frontend.
> 
> Add ASV benchmarks for mercurial that use contrib/perf.py extension that could
> be run against multiple reference repositories.
> 
> The benchmark suite now includes revsets from contrib/base-revsets.txt with
> variants, perftags, perfstatus, perfmanifest and perfheads.
> 
> Installation requires asv>=0.2, python-hglib and virtualenv
> 
> This is part of PerformanceTrackingSuitePlan
> https://www.mercurial-scm.org/wiki/PerformanceTrackingSuitePlan 
> 
> diff --git a/.hgignore b/.hgignore
> --- a/.hgignore
> +++ b/.hgignore
> @@ -49,6 +49,7 @@ mercurial.egg-info
>  tags
>  cscope.*
>  .idea/*
> +.asv/*
>  i18n/hg.pot
>  locale/*/LC_MESSAGES/hg.mo
>  hgext/__index__.py
> diff --git a/contrib/asv.conf.json b/contrib/asv.conf.json
> new file mode 100644
> --- /dev/null
> +++ b/contrib/asv.conf.json
> @@ -0,0 +1,127 @@
> +{
> +    // The version of the config file format.  Do not change, unless
> +    // you know what you are doing.
> +    "version": 1,
> +
> +    // The name of the project being benchmarked
> +    "project": "mercurial",
> +
> +    // The project's homepage
> +    "project_url": "http://mercurial-scm.org/ ",
> +
> +    // The URL or local path of the source code repository for the
> +    // project being benchmarked
> +    "repo": "..",
> +
> +    // List of branches to benchmark. If not provided, defaults to "master"
> +    // (for git) or "default" (for mercurial).
> +    // "branches": ["master"], // for git
> +    // "branches": ["default"],    // for mercurial
> +    "branches": ["default", "stable"],
> +
> +    // The DVCS being used.  If not set, it will be automatically
> +    // determined from "repo" by looking at the protocol in the URL
> +    // (if remote), or by looking for special directories, such as
> +    // ".git" (if local).
> +    // "dvcs": "git",
> +
> +    // The tool to use to create environments.  May be "conda",
> +    // "virtualenv" or other value depending on the plugins in use.
> +    // If missing or the empty string, the tool will be automatically
> +    // determined by looking for tools on the PATH environment
> +    // variable.
> +    "environment_type": "virtualenv",
> +
> +    // the base URL to show a commit for the project.
> +    "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/ ",
> +
> +    // The Pythons you'd like to test against.  If not provided, defaults
> +    // to the current version of Python used to run `asv`.
> +    // "pythons": ["2.7", "3.3"],
> +
> +    // The matrix of dependencies to test.  Each key is the name of a
> +    // package (in PyPI) and the values are version numbers.  An empty
> +    // list or empty string indicates to just test against the default
> +    // (latest) version. null indicates that the package is to not be
> +    // installed. If the package to be tested is only available from
> +    // PyPi, and the 'environment_type' is conda, then you can preface
> +    // the package name by 'pip+', and the package will be installed via
> +    // pip (with all the conda available packages installed first,
> +    // followed by the pip installed packages).
> +    //
> +    // "matrix": {
> +    //     "numpy": ["1.6", "1.7"],
> +    //     "six": ["", null],        // test with and without six installed
> +    //     "pip+emcee": [""],   // emcee is only available for install with pip.
> +    // },
> +
> +    // Combinations of libraries/python versions can be excluded/included
> +    // from the set to test. Each entry is a dictionary containing additional
> +    // key-value pairs to include/exclude.
> +    //
> +    // An exclude entry excludes entries where all values match. The
> +    // values are regexps that should match the whole string.
> +    //
> +    // An include entry adds an environment. Only the packages listed
> +    // are installed. The 'python' key is required. The exclude rules
> +    // do not apply to includes.
> +    //
> +    // In addition to package names, the following keys are available:
> +    //
> +    // - python
> +    //     Python version, as in the *pythons* variable above.
> +    // - environment_type
> +    //     Environment type, as above.
> +    // - sys_platform
> +    //     Platform, as in sys.platform. Possible values for the common
> +    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
> +    //
> +    // "exclude": [
> +    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
> +    //     {"environment_type": "conda", "six": null}, // don't run without six on conda
> +    // ],
> +    //
> +    // "include": [
> +    //     // additional env for python2.7
> +    //     {"python": "2.7", "numpy": "1.8"},
> +    //     // additional env if run on windows+conda
> +    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
> +    // ],
> +
> +    // The directory (relative to the current directory) that benchmarks are
> +    // stored in.  If not provided, defaults to "benchmarks"
> +    "benchmark_dir": "benchmarks",
> +
> +    // The directory (relative to the current directory) to cache the Python
> +    // environments in.  If not provided, defaults to "env"
> +    "env_dir": "../.asv/env",
> +
> +    // The directory (relative to the current directory) that raw benchmark
> +    // results are stored in.  If not provided, defaults to "results".
> +    "results_dir": "../.asv/results",
> +
> +    // The directory (relative to the current directory) that the html tree
> +    // should be written to.  If not provided, defaults to "html".
> +    "html_dir": "../.asv/html",
> +
> +    // The number of characters to retain in the commit hashes.
> +    // "hash_length": 8,
> +
> +    // `asv` will cache wheels of the recent builds in each
> +    // environment, making them faster to install next time.  This is
> +    // number of builds to keep, per environment.
> +    // "wheel_cache_size": 0
> +
> +    // The commits after which the regression search in `asv publish`
> +    // should start looking for regressions. Dictionary whose keys are
> +    // regexps matching to benchmark names, and values corresponding to
> +    // the commit (exclusive) after which to start looking for
> +    // regressions.  The default is to start from the first commit
> +    // with results. If the commit is `null`, regression detection is
> +    // skipped for the matching benchmark.
> +    //
> +    // "regressions_first_commits": {
> +    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
> +    //    "another_benchmark": null,   // Skip regression detection altogether
> +    // }
> +}
> diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/__init__.py
> @@ -0,0 +1,106 @@
> +"""ASV (https://asv.readthedocs.io ) benchmark suite
> +
> +Benchmark are parameterized against reference repositories found in the
> +directory pointed by the REPOS_DIR environment variable.
> +
> +Invocation example:
> +
> +    $ export REPOS_DIR=~/hgperf/repos
> +    # run suite on given revision
> +    $ asv --config contrib/asv.conf.json run REV
> +    # run suite on new changesets found in stable and default branch
> +    $ asv --config contrib/asv.conf.json run NEW
> +    # display a comparative result table of benchmark results between two given
> +    # revisions
> +    $ asv --config contrib/asv.conf.json compare REV1 REV2


> +    # compute regression detection and generate ASV static website
> +    $ asv --config contrib/asv.conf.json publish
> +    # serve the static website
> +    $ asv --config contrib/asv.conf.json preview
> +"""
> +
> +from __future__ import absolute_import
> +
> +import contextlib
> +import functools
> +import io
> +import os
> +import re
> +
> +from mercurial import (
> +    extensions,
> +    hg,
> +    ui as uimod,
> +)
> +
> +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),
> +                          os.path.pardir, os.path.pardir))
> +reposdir = os.environ['REPOS_DIR']
> +reposnames = [name for name in os.listdir(reposdir)
> +              if os.path.isdir(os.path.join(reposdir, name, ".hg"))]

Could you add a check to abort if reposnames is empty?

It took me a lot of time debugging here and into asv code to realize "asv
... compare R1 R2" failing with " "Did not find results for commit R1" means
"The results of R1 are empty", which is actually caused by reposnames here
being empty.

> +outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
> +                       r'\d+.\d+ \(best of \d+\)'))
> +
> +@contextlib.contextmanager
> +def redirectuioutput(ui):
> +    old = ui.fout, ui.ferr
> +    new = io.BytesIO()
> +    try:
> +        ui.fout = ui.ferr = new
> +        yield new
> +    finally:
> +        ui.fout, ui.ferr = old

I think we add a check in change perf.py so that if ui._buffers is not
empty, do not copy or set fout=ferr. Then you can use pushbuffer API here.

> +
> +def runperfcommand(reponame, command, *args, **kwargs):
> +    os.environ["HGRCPATH"] = ""

If people do want to benchmark with some perf-related extensions enabled,
they want customization here. Maybe make it customizable:

  os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", '')

For example, this may be helpful if we want to enable lz4revlog internally.

> +    ui = uimod.ui()
> +    repo = hg.repository(ui, os.path.join(reposdir, reponame))
> +    perfext = extensions.load(ui, 'perfext',
> +                              os.path.join(basedir, 'contrib', 'perf.py'))
> +    cmd = getattr(perfext, command)
> +    with redirectuioutput(ui) as buf:
> +        cmd(ui, repo, *args, **kwargs)
> +    buf.seek(0)
> +    output = buf.read()
> +    match = outputre.search(output)
> +    if not match:
> +        raise ValueError("Invalid output {0}".format(output))
> +    return float(match.group(1))
> +
> +def perfbench(repos=reposnames, name=None, params=None):
> +    """decorator to declare ASV benchmark based on contrib/perf.py extension
> +
> +    An ASV benchmark is a python function with the given attributes:
> +
> +    __name__: should start with track_, time_ or mem_ to be collected by ASV
> +    params and param_name: parameter matrix to display multiple graphs on the
> +    same page.
> +    pretty_name: If defined it's displayed in web-ui instead of __name__
> +    (useful for revsets)
> +    the module name is prepended to the benchmark name and displayed as
> +    "category" in webui.
> +
> +    Benchmarks are automatically parameterized with repositories found in the
> +    REPOS_DIR environment variable.
> +
> +    `params` is the param matrix in the form of a list of tuple
> +    (param_name, [value0, value1])
> +
> +    For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
> +    (a, c), (a, d), (b, c) and (b, d).
> +    """
> +    params = list(params or [])
> +    params.insert(0, ("repo", repos))
> +
> +    def decorator(func):
> +        @functools.wraps(func)
> +        def wrapped(repo, *args):
> +            def perf(command, *a, **kw):
> +                return runperfcommand(repo, command, *a, **kw)
> +            return func(perf, *args)
> +
> +        wrapped.params = [p[1] for p in params]
> +        wrapped.param_names = [p[0] for p in params]
> +        wrapped.pretty_name = name
> +        return wrapped
> +    return decorator
> diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/perf.py
> @@ -0,0 +1,19 @@

Maybe worthwhile adding some comments here saying this file is different
from contrib/perf.py and reuses methods there.

> +from __future__ import absolute_import
> +
> +from . import perfbench
> +
> +@perfbench()
> +def track_tags(perf):
> +    return perf("perftags")
> +
> +@perfbench()
> +def track_status(perf):
> +    return perf("perfstatus", unknown=False)
> +
> +@perfbench(params=[('rev', ['1000', '10000', 'tip'])])
> +def track_manifest(perf, rev):
> +    return perf("perfmanifest", rev)
> +
> +@perfbench()
> +def track_heads(perf):
> +    return perf("perfheads")
> diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py
> new file mode 100644
> --- /dev/null
> +++ b/contrib/benchmarks/revset.py
> @@ -0,0 +1,45 @@
> +"""ASV revset benchmarks generated from contrib/base-revsets.txt
> +
> +Each revset benchmark is parameterized with variants (first, last, sort, ...)
> +"""
> +from __future__ import absolute_import
> +
> +import os
> +import string
> +import sys
> +
> +from . import basedir, perfbench
> +
> +def createrevsetbenchmark(baseset, variants=None):
> +    if variants is None:
> +        # Default variants
> +        variants = ["plain", "first", "last", "sort", "sort+first",
> +                    "sort+last"]
> +    fname = "track_" + "_".join("".join([
> +        c if c in string.digits + string.letters else " "
> +        for c in baseset
> +    ]).split())
> +
> +    def wrap(fname, baseset):
> +        @perfbench(name=baseset, params=[("variant", variants)])
> +        def f(perf, variant):
> +            revset = baseset
> +            if variant != "plain":
> +                for var in variant.split("+"):
> +                    revset = "%s(%s)" % (var, revset)
> +            return perf("perfrevset", revset)
> +        f.__name__ = fname
> +        return f
> +    return wrap(fname, baseset)
> +
> +def initializerevsetbenchmarks():
> +    mod = sys.modules[__name__]
> +    with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'),
> +              'rb') as fh:
> +        for line in fh:
> +            baseset = line.strip()
> +            if baseset and not baseset.startswith('#'):
> +                func = createrevsetbenchmark(baseset)
> +                setattr(mod, func.__name__, func)
> +
> +initializerevsetbenchmarks()
Philippe Pepiot - Nov. 15, 2016, 4:27 p.m.
On 11/14/2016 11:02 AM, Jun Wu wrote:

> Generally looks good to me. Some inline comments about small issues.
>
> I wonder if it is possible to get rid of the dependencies on virtualenv and
> hglib as installing them could be troublesome in some environments. I think
> "virtualenv" could be replaced by just copying the checked out files to
> somewhere, and hglib seems redundant.
Thanks, sent a V3 with requested changes.

Currently asv only support virtualenv (+pip) and conda environments to 
install target package. It could be enhanced to support installation 
using `setup.py install --prefix` and set PYTHONPATH accordingly 
(assuming the package requirements are satisfied).
hglib is required to interact with "repo" from asv.conf.json, it could 
directly use "hg" command via subprocess instead (like git support) but 
I'm not sure it will be relevant from asv point of view.

Cheers,

> Excerpts from Philippe Pepiot's message of 2016-11-03 16:58:13 +0100:
>> # HG changeset patch
>> # User Philippe Pepiot <philippe.pepiot@logilab.fr>
>> # Date 1475136994 -7200
>> #      Thu Sep 29 10:16:34 2016 +0200
>> # Node ID 67e096cea548a37ba80ddf04e62a1cc1d50e9c96
>> # Parent  b032a7b676c6637b2ac6f3ef29431013b15a08f9
>> perf: add asv benchmarks
>>
>> Airspeed velocity (ASV) is a python framework for benchmarking Python packages
>> over their lifetime. The results are displayed in an interactive web frontend.
>>
>> Add ASV benchmarks for mercurial that use contrib/perf.py extension that could
>> be run against multiple reference repositories.
>>
>> The benchmark suite now includes revsets from contrib/base-revsets.txt with
>> variants, perftags, perfstatus, perfmanifest and perfheads.
>>
>> Installation requires asv>=0.2, python-hglib and virtualenv
>>
>> This is part of PerformanceTrackingSuitePlan
>> https://www.mercurial-scm.org/wiki/PerformanceTrackingSuitePlan
>>
>> diff --git a/.hgignore b/.hgignore
>> --- a/.hgignore
>> +++ b/.hgignore
>> @@ -49,6 +49,7 @@ mercurial.egg-info
>>   tags
>>   cscope.*
>>   .idea/*
>> +.asv/*
>>   i18n/hg.pot
>>   locale/*/LC_MESSAGES/hg.mo
>>   hgext/__index__.py
>> diff --git a/contrib/asv.conf.json b/contrib/asv.conf.json
>> new file mode 100644
>> --- /dev/null
>> +++ b/contrib/asv.conf.json
>> @@ -0,0 +1,127 @@
>> +{
>> +    // The version of the config file format.  Do not change, unless
>> +    // you know what you are doing.
>> +    "version": 1,
>> +
>> +    // The name of the project being benchmarked
>> +    "project": "mercurial",
>> +
>> +    // The project's homepage
>> +    "project_url": "http://mercurial-scm.org/ ",
>> +
>> +    // The URL or local path of the source code repository for the
>> +    // project being benchmarked
>> +    "repo": "..",
>> +
>> +    // List of branches to benchmark. If not provided, defaults to "master"
>> +    // (for git) or "default" (for mercurial).
>> +    // "branches": ["master"], // for git
>> +    // "branches": ["default"],    // for mercurial
>> +    "branches": ["default", "stable"],
>> +
>> +    // The DVCS being used.  If not set, it will be automatically
>> +    // determined from "repo" by looking at the protocol in the URL
>> +    // (if remote), or by looking for special directories, such as
>> +    // ".git" (if local).
>> +    // "dvcs": "git",
>> +
>> +    // The tool to use to create environments.  May be "conda",
>> +    // "virtualenv" or other value depending on the plugins in use.
>> +    // If missing or the empty string, the tool will be automatically
>> +    // determined by looking for tools on the PATH environment
>> +    // variable.
>> +    "environment_type": "virtualenv",
>> +
>> +    // the base URL to show a commit for the project.
>> +    "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/ ",
>> +
>> +    // The Pythons you'd like to test against.  If not provided, defaults
>> +    // to the current version of Python used to run `asv`.
>> +    // "pythons": ["2.7", "3.3"],
>> +
>> +    // The matrix of dependencies to test.  Each key is the name of a
>> +    // package (in PyPI) and the values are version numbers.  An empty
>> +    // list or empty string indicates to just test against the default
>> +    // (latest) version. null indicates that the package is to not be
>> +    // installed. If the package to be tested is only available from
>> +    // PyPi, and the 'environment_type' is conda, then you can preface
>> +    // the package name by 'pip+', and the package will be installed via
>> +    // pip (with all the conda available packages installed first,
>> +    // followed by the pip installed packages).
>> +    //
>> +    // "matrix": {
>> +    //     "numpy": ["1.6", "1.7"],
>> +    //     "six": ["", null],        // test with and without six installed
>> +    //     "pip+emcee": [""],   // emcee is only available for install with pip.
>> +    // },
>> +
>> +    // Combinations of libraries/python versions can be excluded/included
>> +    // from the set to test. Each entry is a dictionary containing additional
>> +    // key-value pairs to include/exclude.
>> +    //
>> +    // An exclude entry excludes entries where all values match. The
>> +    // values are regexps that should match the whole string.
>> +    //
>> +    // An include entry adds an environment. Only the packages listed
>> +    // are installed. The 'python' key is required. The exclude rules
>> +    // do not apply to includes.
>> +    //
>> +    // In addition to package names, the following keys are available:
>> +    //
>> +    // - python
>> +    //     Python version, as in the *pythons* variable above.
>> +    // - environment_type
>> +    //     Environment type, as above.
>> +    // - sys_platform
>> +    //     Platform, as in sys.platform. Possible values for the common
>> +    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
>> +    //
>> +    // "exclude": [
>> +    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
>> +    //     {"environment_type": "conda", "six": null}, // don't run without six on conda
>> +    // ],
>> +    //
>> +    // "include": [
>> +    //     // additional env for python2.7
>> +    //     {"python": "2.7", "numpy": "1.8"},
>> +    //     // additional env if run on windows+conda
>> +    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
>> +    // ],
>> +
>> +    // The directory (relative to the current directory) that benchmarks are
>> +    // stored in.  If not provided, defaults to "benchmarks"
>> +    "benchmark_dir": "benchmarks",
>> +
>> +    // The directory (relative to the current directory) to cache the Python
>> +    // environments in.  If not provided, defaults to "env"
>> +    "env_dir": "../.asv/env",
>> +
>> +    // The directory (relative to the current directory) that raw benchmark
>> +    // results are stored in.  If not provided, defaults to "results".
>> +    "results_dir": "../.asv/results",
>> +
>> +    // The directory (relative to the current directory) that the html tree
>> +    // should be written to.  If not provided, defaults to "html".
>> +    "html_dir": "../.asv/html",
>> +
>> +    // The number of characters to retain in the commit hashes.
>> +    // "hash_length": 8,
>> +
>> +    // `asv` will cache wheels of the recent builds in each
>> +    // environment, making them faster to install next time.  This is
>> +    // number of builds to keep, per environment.
>> +    // "wheel_cache_size": 0
>> +
>> +    // The commits after which the regression search in `asv publish`
>> +    // should start looking for regressions. Dictionary whose keys are
>> +    // regexps matching to benchmark names, and values corresponding to
>> +    // the commit (exclusive) after which to start looking for
>> +    // regressions.  The default is to start from the first commit
>> +    // with results. If the commit is `null`, regression detection is
>> +    // skipped for the matching benchmark.
>> +    //
>> +    // "regressions_first_commits": {
>> +    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
>> +    //    "another_benchmark": null,   // Skip regression detection altogether
>> +    // }
>> +}
>> diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py
>> new file mode 100644
>> --- /dev/null
>> +++ b/contrib/benchmarks/__init__.py
>> @@ -0,0 +1,106 @@
>> +"""ASV (https://asv.readthedocs.io ) benchmark suite
>> +
>> +Benchmark are parameterized against reference repositories found in the
>> +directory pointed by the REPOS_DIR environment variable.
>> +
>> +Invocation example:
>> +
>> +    $ export REPOS_DIR=~/hgperf/repos
>> +    # run suite on given revision
>> +    $ asv --config contrib/asv.conf.json run REV
>> +    # run suite on new changesets found in stable and default branch
>> +    $ asv --config contrib/asv.conf.json run NEW
>> +    # display a comparative result table of benchmark results between two given
>> +    # revisions
>> +    $ asv --config contrib/asv.conf.json compare REV1 REV2
>
>> +    # compute regression detection and generate ASV static website
>> +    $ asv --config contrib/asv.conf.json publish
>> +    # serve the static website
>> +    $ asv --config contrib/asv.conf.json preview
>> +"""
>> +
>> +from __future__ import absolute_import
>> +
>> +import contextlib
>> +import functools
>> +import io
>> +import os
>> +import re
>> +
>> +from mercurial import (
>> +    extensions,
>> +    hg,
>> +    ui as uimod,
>> +)
>> +
>> +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),
>> +                          os.path.pardir, os.path.pardir))
>> +reposdir = os.environ['REPOS_DIR']
>> +reposnames = [name for name in os.listdir(reposdir)
>> +              if os.path.isdir(os.path.join(reposdir, name, ".hg"))]
> Could you add a check to abort if reposnames is empty?
>
> It took me a lot of time debugging here and into asv code to realize "asv
> ... compare R1 R2" failing with " "Did not find results for commit R1" means
> "The results of R1 are empty", which is actually caused by reposnames here
> being empty.
>
>> +outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
>> +                       r'\d+.\d+ \(best of \d+\)'))
>> +
>> +@contextlib.contextmanager
>> +def redirectuioutput(ui):
>> +    old = ui.fout, ui.ferr
>> +    new = io.BytesIO()
>> +    try:
>> +        ui.fout = ui.ferr = new
>> +        yield new
>> +    finally:
>> +        ui.fout, ui.ferr = old
> I think we add a check in change perf.py so that if ui._buffers is not
> empty, do not copy or set fout=ferr. Then you can use pushbuffer API here.
>
>> +
>> +def runperfcommand(reponame, command, *args, **kwargs):
>> +    os.environ["HGRCPATH"] = ""
> If people do want to benchmark with some perf-related extensions enabled,
> they want customization here. Maybe make it customizable:
>
>    os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", '')
>
> For example, this may be helpful if we want to enable lz4revlog internally.
>
>> +    ui = uimod.ui()
>> +    repo = hg.repository(ui, os.path.join(reposdir, reponame))
>> +    perfext = extensions.load(ui, 'perfext',
>> +                              os.path.join(basedir, 'contrib', 'perf.py'))
>> +    cmd = getattr(perfext, command)
>> +    with redirectuioutput(ui) as buf:
>> +        cmd(ui, repo, *args, **kwargs)
>> +    buf.seek(0)
>> +    output = buf.read()
>> +    match = outputre.search(output)
>> +    if not match:
>> +        raise ValueError("Invalid output {0}".format(output))
>> +    return float(match.group(1))
>> +
>> +def perfbench(repos=reposnames, name=None, params=None):
>> +    """decorator to declare ASV benchmark based on contrib/perf.py extension
>> +
>> +    An ASV benchmark is a python function with the given attributes:
>> +
>> +    __name__: should start with track_, time_ or mem_ to be collected by ASV
>> +    params and param_name: parameter matrix to display multiple graphs on the
>> +    same page.
>> +    pretty_name: If defined it's displayed in web-ui instead of __name__
>> +    (useful for revsets)
>> +    the module name is prepended to the benchmark name and displayed as
>> +    "category" in webui.
>> +
>> +    Benchmarks are automatically parameterized with repositories found in the
>> +    REPOS_DIR environment variable.
>> +
>> +    `params` is the param matrix in the form of a list of tuple
>> +    (param_name, [value0, value1])
>> +
>> +    For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
>> +    (a, c), (a, d), (b, c) and (b, d).
>> +    """
>> +    params = list(params or [])
>> +    params.insert(0, ("repo", repos))
>> +
>> +    def decorator(func):
>> +        @functools.wraps(func)
>> +        def wrapped(repo, *args):
>> +            def perf(command, *a, **kw):
>> +                return runperfcommand(repo, command, *a, **kw)
>> +            return func(perf, *args)
>> +
>> +        wrapped.params = [p[1] for p in params]
>> +        wrapped.param_names = [p[0] for p in params]
>> +        wrapped.pretty_name = name
>> +        return wrapped
>> +    return decorator
>> diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py
>> new file mode 100644
>> --- /dev/null
>> +++ b/contrib/benchmarks/perf.py
>> @@ -0,0 +1,19 @@
> Maybe worthwhile adding some comments here saying this file is different
> from contrib/perf.py and reuses methods there.
>
>> +from __future__ import absolute_import
>> +
>> +from . import perfbench
>> +
>> +@perfbench()
>> +def track_tags(perf):
>> +    return perf("perftags")
>> +
>> +@perfbench()
>> +def track_status(perf):
>> +    return perf("perfstatus", unknown=False)
>> +
>> +@perfbench(params=[('rev', ['1000', '10000', 'tip'])])
>> +def track_manifest(perf, rev):
>> +    return perf("perfmanifest", rev)
>> +
>> +@perfbench()
>> +def track_heads(perf):
>> +    return perf("perfheads")
>> diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py
>> new file mode 100644
>> --- /dev/null
>> +++ b/contrib/benchmarks/revset.py
>> @@ -0,0 +1,45 @@
>> +"""ASV revset benchmarks generated from contrib/base-revsets.txt
>> +
>> +Each revset benchmark is parameterized with variants (first, last, sort, ...)
>> +"""
>> +from __future__ import absolute_import
>> +
>> +import os
>> +import string
>> +import sys
>> +
>> +from . import basedir, perfbench
>> +
>> +def createrevsetbenchmark(baseset, variants=None):
>> +    if variants is None:
>> +        # Default variants
>> +        variants = ["plain", "first", "last", "sort", "sort+first",
>> +                    "sort+last"]
>> +    fname = "track_" + "_".join("".join([
>> +        c if c in string.digits + string.letters else " "
>> +        for c in baseset
>> +    ]).split())
>> +
>> +    def wrap(fname, baseset):
>> +        @perfbench(name=baseset, params=[("variant", variants)])
>> +        def f(perf, variant):
>> +            revset = baseset
>> +            if variant != "plain":
>> +                for var in variant.split("+"):
>> +                    revset = "%s(%s)" % (var, revset)
>> +            return perf("perfrevset", revset)
>> +        f.__name__ = fname
>> +        return f
>> +    return wrap(fname, baseset)
>> +
>> +def initializerevsetbenchmarks():
>> +    mod = sys.modules[__name__]
>> +    with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'),
>> +              'rb') as fh:
>> +        for line in fh:
>> +            baseset = line.strip()
>> +            if baseset and not baseset.startswith('#'):
>> +                func = createrevsetbenchmark(baseset)
>> +                setattr(mod, func.__name__, func)
>> +
>> +initializerevsetbenchmarks()

Patch

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -49,6 +49,7 @@  mercurial.egg-info
 tags
 cscope.*
 .idea/*
+.asv/*
 i18n/hg.pot
 locale/*/LC_MESSAGES/hg.mo
 hgext/__index__.py
diff --git a/contrib/asv.conf.json b/contrib/asv.conf.json
new file mode 100644
--- /dev/null
+++ b/contrib/asv.conf.json
@@ -0,0 +1,127 @@ 
+{
+    // The version of the config file format.  Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+
+    // The name of the project being benchmarked
+    "project": "mercurial",
+
+    // The project's homepage
+    "project_url": "http://mercurial-scm.org/",
+
+    // The URL or local path of the source code repository for the
+    // project being benchmarked
+    "repo": "..",
+
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "default" (for mercurial).
+    // "branches": ["master"], // for git
+    // "branches": ["default"],    // for mercurial
+    "branches": ["default", "stable"],
+
+    // The DVCS being used.  If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    // "dvcs": "git",
+
+    // The tool to use to create environments.  May be "conda",
+    // "virtualenv" or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "virtualenv",
+
+    // the base URL to show a commit for the project.
+    "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/",
+
+    // The Pythons you'd like to test against.  If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    // "pythons": ["2.7", "3.3"],
+
+    // The matrix of dependencies to test.  Each key is the name of a
+    // package (in PyPI) and the values are version numbers.  An empty
+    // list or empty string indicates to just test against the default
+    // (latest) version. null indicates that the package is to not be
+    // installed. If the package to be tested is only available from
+    // PyPi, and the 'environment_type' is conda, then you can preface
+    // the package name by 'pip+', and the package will be installed via
+    // pip (with all the conda available packages installed first,
+    // followed by the pip installed packages).
+    //
+    // "matrix": {
+    //     "numpy": ["1.6", "1.7"],
+    //     "six": ["", null],        // test with and without six installed
+    //     "pip+emcee": [""],   // emcee is only available for install with pip.
+    // },
+
+    // Combinations of libraries/python versions can be excluded/included
+    // from the set to test. Each entry is a dictionary containing additional
+    // key-value pairs to include/exclude.
+    //
+    // An exclude entry excludes entries where all values match. The
+    // values are regexps that should match the whole string.
+    //
+    // An include entry adds an environment. Only the packages listed
+    // are installed. The 'python' key is required. The exclude rules
+    // do not apply to includes.
+    //
+    // In addition to package names, the following keys are available:
+    //
+    // - python
+    //     Python version, as in the *pythons* variable above.
+    // - environment_type
+    //     Environment type, as above.
+    // - sys_platform
+    //     Platform, as in sys.platform. Possible values for the common
+    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
+    //
+    // "exclude": [
+    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
+    //     {"environment_type": "conda", "six": null}, // don't run without six on conda
+    // ],
+    //
+    // "include": [
+    //     // additional env for python2.7
+    //     {"python": "2.7", "numpy": "1.8"},
+    //     // additional env if run on windows+conda
+    //     {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""},
+    // ],
+
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in.  If not provided, defaults to "benchmarks"
+    "benchmark_dir": "benchmarks",
+
+    // The directory (relative to the current directory) to cache the Python
+    // environments in.  If not provided, defaults to "env"
+    "env_dir": "../.asv/env",
+
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in.  If not provided, defaults to "results".
+    "results_dir": "../.asv/results",
+
+    // The directory (relative to the current directory) that the html tree
+    // should be written to.  If not provided, defaults to "html".
+    "html_dir": "../.asv/html",
+
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+
+    // `asv` will cache wheels of the recent builds in each
+    // environment, making them faster to install next time.  This is
+    // number of builds to keep, per environment.
+    // "wheel_cache_size": 0
+
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions.  The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // }
+}
diff --git a/contrib/benchmarks/__init__.py b/contrib/benchmarks/__init__.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/__init__.py
@@ -0,0 +1,106 @@ 
+"""ASV (https://asv.readthedocs.io) benchmark suite
+
+Benchmark are parameterized against reference repositories found in the
+directory pointed by the REPOS_DIR environment variable.
+
+Invocation example:
+
+    $ export REPOS_DIR=~/hgperf/repos
+    # run suite on given revision
+    $ asv --config contrib/asv.conf.json run REV
+    # run suite on new changesets found in stable and default branch
+    $ asv --config contrib/asv.conf.json run NEW
+    # display a comparative result table of benchmark results between two given
+    # revisions
+    $ asv --config contrib/asv.conf.json compare REV1 REV2
+    # compute regression detection and generate ASV static website
+    $ asv --config contrib/asv.conf.json publish
+    # serve the static website
+    $ asv --config contrib/asv.conf.json preview
+"""
+
+from __future__ import absolute_import
+
+import contextlib
+import functools
+import io
+import os
+import re
+
+from mercurial import (
+    extensions,
+    hg,
+    ui as uimod,
+)
+
+basedir = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                          os.path.pardir, os.path.pardir))
+reposdir = os.environ['REPOS_DIR']
+reposnames = [name for name in os.listdir(reposdir)
+              if os.path.isdir(os.path.join(reposdir, name, ".hg"))]
+outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys '
+                       r'\d+.\d+ \(best of \d+\)'))
+
+@contextlib.contextmanager
+def redirectuioutput(ui):
+    old = ui.fout, ui.ferr
+    new = io.BytesIO()
+    try:
+        ui.fout = ui.ferr = new
+        yield new
+    finally:
+        ui.fout, ui.ferr = old
+
+def runperfcommand(reponame, command, *args, **kwargs):
+    os.environ["HGRCPATH"] = ""
+    ui = uimod.ui()
+    repo = hg.repository(ui, os.path.join(reposdir, reponame))
+    perfext = extensions.load(ui, 'perfext',
+                              os.path.join(basedir, 'contrib', 'perf.py'))
+    cmd = getattr(perfext, command)
+    with redirectuioutput(ui) as buf:
+        cmd(ui, repo, *args, **kwargs)
+    buf.seek(0)
+    output = buf.read()
+    match = outputre.search(output)
+    if not match:
+        raise ValueError("Invalid output {0}".format(output))
+    return float(match.group(1))
+
+def perfbench(repos=reposnames, name=None, params=None):
+    """decorator to declare ASV benchmark based on contrib/perf.py extension
+
+    An ASV benchmark is a python function with the given attributes:
+
+    __name__: should start with track_, time_ or mem_ to be collected by ASV
+    params and param_name: parameter matrix to display multiple graphs on the
+    same page.
+    pretty_name: If defined it's displayed in web-ui instead of __name__
+    (useful for revsets)
+    the module name is prepended to the benchmark name and displayed as
+    "category" in webui.
+
+    Benchmarks are automatically parameterized with repositories found in the
+    REPOS_DIR environment variable.
+
+    `params` is the param matrix in the form of a list of tuple
+    (param_name, [value0, value1])
+
+    For example [(x, [a, b]), (y, [c, d])] declare benchmarks for
+    (a, c), (a, d), (b, c) and (b, d).
+    """
+    params = list(params or [])
+    params.insert(0, ("repo", repos))
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapped(repo, *args):
+            def perf(command, *a, **kw):
+                return runperfcommand(repo, command, *a, **kw)
+            return func(perf, *args)
+
+        wrapped.params = [p[1] for p in params]
+        wrapped.param_names = [p[0] for p in params]
+        wrapped.pretty_name = name
+        return wrapped
+    return decorator
diff --git a/contrib/benchmarks/perf.py b/contrib/benchmarks/perf.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/perf.py
@@ -0,0 +1,19 @@ 
+from __future__ import absolute_import
+
+from . import perfbench
+
+@perfbench()
+def track_tags(perf):
+    return perf("perftags")
+
+@perfbench()
+def track_status(perf):
+    return perf("perfstatus", unknown=False)
+
+@perfbench(params=[('rev', ['1000', '10000', 'tip'])])
+def track_manifest(perf, rev):
+    return perf("perfmanifest", rev)
+
+@perfbench()
+def track_heads(perf):
+    return perf("perfheads")
diff --git a/contrib/benchmarks/revset.py b/contrib/benchmarks/revset.py
new file mode 100644
--- /dev/null
+++ b/contrib/benchmarks/revset.py
@@ -0,0 +1,45 @@ 
+"""ASV revset benchmarks generated from contrib/base-revsets.txt
+
+Each revset benchmark is parameterized with variants (first, last, sort, ...)
+"""
+from __future__ import absolute_import
+
+import os
+import string
+import sys
+
+from . import basedir, perfbench
+
+def createrevsetbenchmark(baseset, variants=None):
+    if variants is None:
+        # Default variants
+        variants = ["plain", "first", "last", "sort", "sort+first",
+                    "sort+last"]
+    fname = "track_" + "_".join("".join([
+        c if c in string.digits + string.letters else " "
+        for c in baseset
+    ]).split())
+
+    def wrap(fname, baseset):
+        @perfbench(name=baseset, params=[("variant", variants)])
+        def f(perf, variant):
+            revset = baseset
+            if variant != "plain":
+                for var in variant.split("+"):
+                    revset = "%s(%s)" % (var, revset)
+            return perf("perfrevset", revset)
+        f.__name__ = fname
+        return f
+    return wrap(fname, baseset)
+
+def initializerevsetbenchmarks():
+    mod = sys.modules[__name__]
+    with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'),
+              'rb') as fh:
+        for line in fh:
+            baseset = line.strip()
+            if baseset and not baseset.startswith('#'):
+                func = createrevsetbenchmark(baseset)
+                setattr(mod, func.__name__, func)
+
+initializerevsetbenchmarks()