Patchwork changegroup: add bundlecaps back

login
register
mail settings
Submitter Durham Goode
Date May 9, 2017, 7:59 p.m.
Message ID <0b03e45736e3699ae782.1494359997@dev111.prn1.facebook.com>
Download mbox | patch
Permalink /patch/20543/
State Superseded
Headers show

Comments

Durham Goode - May 9, 2017, 7:59 p.m.
# HG changeset patch
# User Durham Goode <durham@fb.com>
# Date 1494359977 25200
#      Tue May 09 12:59:37 2017 -0700
# Node ID 0b03e45736e3699ae7828d4f7204cae4f079b831
# Parent  8f1a2b848b52ea7bf3fe2404e3b62924c7aae93f
changegroup: add bundlecaps back

Commit 282b288aa20c333c removed the unused bundlecaps argument from the
changegroup code. While it is unused in core Mercurial, it was an important
feature for the remotefilelog extension because it allowed the exchange layer to
communicate to the changegroup packer that this was a shallow repo and that
filelogs should not be included. Without bundlecaps, there is currently no other
way to pass that information along without a more extensive refactor of
exchange, bundle, and changegroup code.

This patch backs out the original removal, and merges it with some recent
changes to changegroup apis.
Augie Fackler - May 13, 2017, 2:18 a.m.
On Tue, May 09, 2017 at 12:59:57PM -0700, Durham Goode wrote:
> # HG changeset patch
> # User Durham Goode <durham@fb.com>
> # Date 1494359977 25200
> #      Tue May 09 12:59:37 2017 -0700
> # Node ID 0b03e45736e3699ae7828d4f7204cae4f079b831
> # Parent  8f1a2b848b52ea7bf3fe2404e3b62924c7aae93f
> changegroup: add bundlecaps back

Ugh, this has fallen behind already. Can you rebase it and resend it?

(I tried rebasing it myself but it wasn't immediately obvious which
side I should take in some conflicts.)

>
> Commit 282b288aa20c333c removed the unused bundlecaps argument from the
> changegroup code. While it is unused in core Mercurial, it was an important
> feature for the remotefilelog extension because it allowed the exchange layer to
> communicate to the changegroup packer that this was a shallow repo and that
> filelogs should not be included. Without bundlecaps, there is currently no other
> way to pass that information along without a more extensive refactor of
> exchange, bundle, and changegroup code.
>
> This patch backs out the original removal, and merges it with some recent
> changes to changegroup apis.
>
> diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
> --- a/mercurial/changegroup.py
> +++ b/mercurial/changegroup.py
> @@ -502,9 +502,18 @@ class headerlessfixup(object):
>  class cg1packer(object):
>      deltaheader = _CHANGEGROUPV1_DELTA_HEADER
>      version = '01'
> -    def __init__(self, repo):
> +    def __init__(self, repo, bundlecaps=None):
>          """Given a source repo, construct a bundler.
> +
> +        bundlecaps is optional and can be used to specify the set of
> +        capabilities which can be used to build the bundle. While bundlecaps is
> +        unused in core Mercurial, extensions rely on this feature to communicate
> +        capabilities to customize the changegroup packer.
>          """
> +        # Set of capabilities we can use to build the bundle.
> +        if bundlecaps is None:
> +            bundlecaps = set()
> +        self._bundlecaps = bundlecaps
>          # experimental config: bundle.reorder
>          reorder = repo.ui.config('bundle', 'reorder', 'auto')
>          if reorder == 'auto':
> @@ -808,8 +817,8 @@ class cg2packer(cg1packer):
>      version = '02'
>      deltaheader = _CHANGEGROUPV2_DELTA_HEADER
>
> -    def __init__(self, repo):
> -        super(cg2packer, self).__init__(repo)
> +    def __init__(self, repo, bundlecaps=None):
> +        super(cg2packer, self).__init__(repo, bundlecaps)
>          if self._reorder is None:
>              # Since generaldelta is directly supported by cg2, reordering
>              # generally doesn't help, so we disable it by default (treating
> @@ -903,9 +912,9 @@ def safeversion(repo):
>      assert versions
>      return min(versions)
>
> -def getbundler(version, repo):
> +def getbundler(version, repo, bundlecaps=None):
>      assert version in supportedoutgoingversions(repo)
> -    return _packermap[version][0](repo)
> +    return _packermap[version][0](repo, bundlecaps)
>
>  def getunbundler(version, fh, alg, extras=None):
>      return _packermap[version][1](fh, alg, extras=extras)
> @@ -956,24 +965,26 @@ def changegroupsubset(repo, roots, heads
>      bundler = getbundler(version, repo)
>      return getsubset(repo, outgoing, bundler, source)
>
> -def getlocalchangegroupraw(repo, source, outgoing, version='01'):
> +def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
> +                           version='01'):
>      """Like getbundle, but taking a discovery.outgoing as an argument.
>
>      This is only implemented for local repos and reuses potentially
>      precomputed sets in outgoing. Returns a raw changegroup generator."""
>      if not outgoing.missing:
>          return None
> -    bundler = getbundler(version, repo)
> +    bundler = getbundler(version, repo, bundlecaps)
>      return getsubsetraw(repo, outgoing, bundler, source)
>
> -def getchangegroup(repo, source, outgoing, version='01'):
> +def getchangegroup(repo, source, outgoing, bundlecaps=None,
> +                   version='01'):
>      """Like getbundle, but taking a discovery.outgoing as an argument.
>
>      This is only implemented for local repos and reuses potentially
>      precomputed sets in outgoing."""
>      if not outgoing.missing:
>          return None
> -    bundler = getbundler(version, repo)
> +    bundler = getbundler(version, repo, bundlecaps)
>      return getsubset(repo, outgoing, bundler, source)
>
>  def getlocalchangegroup(repo, *args, **kwargs):
> diff --git a/mercurial/commands.py b/mercurial/commands.py
> --- a/mercurial/commands.py
> +++ b/mercurial/commands.py
> @@ -1346,6 +1346,8 @@ def bundle(ui, repo, fname, dest=None, *
>          base = ['null']
>      else:
>          base = scmutil.revrange(repo, opts.get('base'))
> +    # TODO: get desired bundlecaps from command line.
> +    bundlecaps = None
>      if cgversion not in changegroup.supportedoutgoingversions(repo):
>          raise error.Abort(_("repository does not support bundle version %s") %
>                            cgversion)
> @@ -1367,7 +1369,9 @@ def bundle(ui, repo, fname, dest=None, *
>                                                  onlyheads=heads,
>                                                  force=opts.get('force'),
>                                                  portable=True)
> -    cg = changegroup.getchangegroup(repo, 'bundle', outgoing, version=cgversion)
> +    cg = changegroup.getchangegroup(repo, 'bundle', outgoing,
> +                                    bundlecaps=bundlecaps,
> +                                    version=cgversion)
>      if not cg:
>          scmutil.nochangesfound(ui, repo, not base and outgoing.excluded)
>          return 1
> diff --git a/mercurial/exchange.py b/mercurial/exchange.py
> --- a/mercurial/exchange.py
> +++ b/mercurial/exchange.py
> @@ -938,19 +938,22 @@ def _pushchangeset(pushop):
>      pushop.repo.prepushoutgoinghooks(pushop)
>      outgoing = pushop.outgoing
>      unbundle = pushop.remote.capable('unbundle')
> +    # TODO: get bundlecaps from remote
> +    bundlecaps = None
>      # create a changegroup from local
>      if pushop.revs is None and not (outgoing.excluded
>                              or pushop.repo.changelog.filteredrevs):
>          # push everything,
>          # use the fast path, no race possible on push
> -        bundler = changegroup.cg1packer(pushop.repo)
> +        bundler = changegroup.cg1packer(pushop.repo, bundlecaps)
>          cg = changegroup.getsubset(pushop.repo,
>                                     outgoing,
>                                     bundler,
>                                     'push',
>                                     fastpath=True)
>      else:
> -        cg = changegroup.getchangegroup(pushop.repo, 'push', outgoing)
> +        cg = changegroup.getchangegroup(pushop.repo, 'push', outgoing,
> +                                        bundlecaps=bundlecaps)
>
>      # apply changegroup to remote
>      if unbundle:
> @@ -1575,7 +1578,7 @@ def getbundlechunks(repo, source, heads=
>              raise ValueError(_('unsupported getbundle arguments: %s')
>                               % ', '.join(sorted(kwargs.keys())))
>          outgoing = _computeoutgoing(repo, heads, common)
> -        bundler = changegroup.getbundler('01', repo)
> +        bundler = changegroup.getbundler('01', repo, bundlecaps)
>          return changegroup.getsubsetraw(repo, outgoing, bundler, source)
>
>      # bundle20 case
> @@ -1613,6 +1616,7 @@ def _getbundlechangegrouppart(bundler, r
>              version = max(cgversions)
>          outgoing = _computeoutgoing(repo, heads, common)
>          cg = changegroup.getlocalchangegroupraw(repo, source, outgoing,
> +                                                bundlecaps=bundlecaps,
>                                                  version=version)
>
>      if cg:
> diff --git a/tests/test-bundle2-format.t b/tests/test-bundle2-format.t
> --- a/tests/test-bundle2-format.t
> +++ b/tests/test-bundle2-format.t
> @@ -113,7 +113,7 @@ Create an extension to test bundle2 API
>    >             headmissing = [c.node() for c in repo.set('heads(%ld)', revs)]
>    >             headcommon  = [c.node() for c in repo.set('parents(%ld) - %ld', revs, revs)]
>    >             outgoing = discovery.outgoing(repo, headcommon, headmissing)
> -  >             cg = changegroup.getchangegroup(repo, 'test:bundle2', outgoing)
> +  >             cg = changegroup.getchangegroup(repo, 'test:bundle2', outgoing, None)
>    >             bundler.newpart('changegroup', data=cg.getchunks(),
>    >                             mandatory=False)
>    >
> diff --git a/tests/test-bundle2-multiple-changegroups.t b/tests/test-bundle2-multiple-changegroups.t
> --- a/tests/test-bundle2-multiple-changegroups.t
> +++ b/tests/test-bundle2-multiple-changegroups.t
> @@ -13,11 +13,13 @@ Create an extension to test bundle2 with
>    >     # in 'heads' as intermediate heads for the first changegroup.
>    >     intermediates = [repo[r].p1().node() for r in heads]
>    >     outgoing = discovery.outgoing(repo, common, intermediates)
> -  >     cg = changegroup.getchangegroup(repo, source, outgoing)
> +  >     cg = changegroup.getchangegroup(repo, source, outgoing,
> +  >                                     bundlecaps=bundlecaps)
>    >     bundler.newpart('output', data='changegroup1')
>    >     bundler.newpart('changegroup', data=cg.getchunks())
>    >     outgoing = discovery.outgoing(repo, common + intermediates, heads)
> -  >     cg = changegroup.getchangegroup(repo, source, outgoing)
> +  >     cg = changegroup.getchangegroup(repo, source, outgoing,
> +  >                                     bundlecaps=bundlecaps)
>    >     bundler.newpart('output', data='changegroup2')
>    >     bundler.newpart('changegroup', data=cg.getchunks())
>    >
> _______________________________________________
> Mercurial-devel mailing list
> Mercurial-devel@mercurial-scm.org
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

Patch

diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -502,9 +502,18 @@  class headerlessfixup(object):
 class cg1packer(object):
     deltaheader = _CHANGEGROUPV1_DELTA_HEADER
     version = '01'
-    def __init__(self, repo):
+    def __init__(self, repo, bundlecaps=None):
         """Given a source repo, construct a bundler.
+
+        bundlecaps is optional and can be used to specify the set of
+        capabilities which can be used to build the bundle. While bundlecaps is
+        unused in core Mercurial, extensions rely on this feature to communicate
+        capabilities to customize the changegroup packer.
         """
+        # Set of capabilities we can use to build the bundle.
+        if bundlecaps is None:
+            bundlecaps = set()
+        self._bundlecaps = bundlecaps
         # experimental config: bundle.reorder
         reorder = repo.ui.config('bundle', 'reorder', 'auto')
         if reorder == 'auto':
@@ -808,8 +817,8 @@  class cg2packer(cg1packer):
     version = '02'
     deltaheader = _CHANGEGROUPV2_DELTA_HEADER
 
-    def __init__(self, repo):
-        super(cg2packer, self).__init__(repo)
+    def __init__(self, repo, bundlecaps=None):
+        super(cg2packer, self).__init__(repo, bundlecaps)
         if self._reorder is None:
             # Since generaldelta is directly supported by cg2, reordering
             # generally doesn't help, so we disable it by default (treating
@@ -903,9 +912,9 @@  def safeversion(repo):
     assert versions
     return min(versions)
 
-def getbundler(version, repo):
+def getbundler(version, repo, bundlecaps=None):
     assert version in supportedoutgoingversions(repo)
-    return _packermap[version][0](repo)
+    return _packermap[version][0](repo, bundlecaps)
 
 def getunbundler(version, fh, alg, extras=None):
     return _packermap[version][1](fh, alg, extras=extras)
@@ -956,24 +965,26 @@  def changegroupsubset(repo, roots, heads
     bundler = getbundler(version, repo)
     return getsubset(repo, outgoing, bundler, source)
 
-def getlocalchangegroupraw(repo, source, outgoing, version='01'):
+def getlocalchangegroupraw(repo, source, outgoing, bundlecaps=None,
+                           version='01'):
     """Like getbundle, but taking a discovery.outgoing as an argument.
 
     This is only implemented for local repos and reuses potentially
     precomputed sets in outgoing. Returns a raw changegroup generator."""
     if not outgoing.missing:
         return None
-    bundler = getbundler(version, repo)
+    bundler = getbundler(version, repo, bundlecaps)
     return getsubsetraw(repo, outgoing, bundler, source)
 
-def getchangegroup(repo, source, outgoing, version='01'):
+def getchangegroup(repo, source, outgoing, bundlecaps=None,
+                   version='01'):
     """Like getbundle, but taking a discovery.outgoing as an argument.
 
     This is only implemented for local repos and reuses potentially
     precomputed sets in outgoing."""
     if not outgoing.missing:
         return None
-    bundler = getbundler(version, repo)
+    bundler = getbundler(version, repo, bundlecaps)
     return getsubset(repo, outgoing, bundler, source)
 
 def getlocalchangegroup(repo, *args, **kwargs):
diff --git a/mercurial/commands.py b/mercurial/commands.py
--- a/mercurial/commands.py
+++ b/mercurial/commands.py
@@ -1346,6 +1346,8 @@  def bundle(ui, repo, fname, dest=None, *
         base = ['null']
     else:
         base = scmutil.revrange(repo, opts.get('base'))
+    # TODO: get desired bundlecaps from command line.
+    bundlecaps = None
     if cgversion not in changegroup.supportedoutgoingversions(repo):
         raise error.Abort(_("repository does not support bundle version %s") %
                           cgversion)
@@ -1367,7 +1369,9 @@  def bundle(ui, repo, fname, dest=None, *
                                                 onlyheads=heads,
                                                 force=opts.get('force'),
                                                 portable=True)
-    cg = changegroup.getchangegroup(repo, 'bundle', outgoing, version=cgversion)
+    cg = changegroup.getchangegroup(repo, 'bundle', outgoing,
+                                    bundlecaps=bundlecaps,
+                                    version=cgversion)
     if not cg:
         scmutil.nochangesfound(ui, repo, not base and outgoing.excluded)
         return 1
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -938,19 +938,22 @@  def _pushchangeset(pushop):
     pushop.repo.prepushoutgoinghooks(pushop)
     outgoing = pushop.outgoing
     unbundle = pushop.remote.capable('unbundle')
+    # TODO: get bundlecaps from remote
+    bundlecaps = None
     # create a changegroup from local
     if pushop.revs is None and not (outgoing.excluded
                             or pushop.repo.changelog.filteredrevs):
         # push everything,
         # use the fast path, no race possible on push
-        bundler = changegroup.cg1packer(pushop.repo)
+        bundler = changegroup.cg1packer(pushop.repo, bundlecaps)
         cg = changegroup.getsubset(pushop.repo,
                                    outgoing,
                                    bundler,
                                    'push',
                                    fastpath=True)
     else:
-        cg = changegroup.getchangegroup(pushop.repo, 'push', outgoing)
+        cg = changegroup.getchangegroup(pushop.repo, 'push', outgoing,
+                                        bundlecaps=bundlecaps)
 
     # apply changegroup to remote
     if unbundle:
@@ -1575,7 +1578,7 @@  def getbundlechunks(repo, source, heads=
             raise ValueError(_('unsupported getbundle arguments: %s')
                              % ', '.join(sorted(kwargs.keys())))
         outgoing = _computeoutgoing(repo, heads, common)
-        bundler = changegroup.getbundler('01', repo)
+        bundler = changegroup.getbundler('01', repo, bundlecaps)
         return changegroup.getsubsetraw(repo, outgoing, bundler, source)
 
     # bundle20 case
@@ -1613,6 +1616,7 @@  def _getbundlechangegrouppart(bundler, r
             version = max(cgversions)
         outgoing = _computeoutgoing(repo, heads, common)
         cg = changegroup.getlocalchangegroupraw(repo, source, outgoing,
+                                                bundlecaps=bundlecaps,
                                                 version=version)
 
     if cg:
diff --git a/tests/test-bundle2-format.t b/tests/test-bundle2-format.t
--- a/tests/test-bundle2-format.t
+++ b/tests/test-bundle2-format.t
@@ -113,7 +113,7 @@  Create an extension to test bundle2 API
   >             headmissing = [c.node() for c in repo.set('heads(%ld)', revs)]
   >             headcommon  = [c.node() for c in repo.set('parents(%ld) - %ld', revs, revs)]
   >             outgoing = discovery.outgoing(repo, headcommon, headmissing)
-  >             cg = changegroup.getchangegroup(repo, 'test:bundle2', outgoing)
+  >             cg = changegroup.getchangegroup(repo, 'test:bundle2', outgoing, None)
   >             bundler.newpart('changegroup', data=cg.getchunks(),
   >                             mandatory=False)
   > 
diff --git a/tests/test-bundle2-multiple-changegroups.t b/tests/test-bundle2-multiple-changegroups.t
--- a/tests/test-bundle2-multiple-changegroups.t
+++ b/tests/test-bundle2-multiple-changegroups.t
@@ -13,11 +13,13 @@  Create an extension to test bundle2 with
   >     # in 'heads' as intermediate heads for the first changegroup.
   >     intermediates = [repo[r].p1().node() for r in heads]
   >     outgoing = discovery.outgoing(repo, common, intermediates)
-  >     cg = changegroup.getchangegroup(repo, source, outgoing)
+  >     cg = changegroup.getchangegroup(repo, source, outgoing,
+  >                                     bundlecaps=bundlecaps)
   >     bundler.newpart('output', data='changegroup1')
   >     bundler.newpart('changegroup', data=cg.getchunks())
   >     outgoing = discovery.outgoing(repo, common + intermediates, heads)
-  >     cg = changegroup.getchangegroup(repo, source, outgoing)
+  >     cg = changegroup.getchangegroup(repo, source, outgoing,
+  >                                     bundlecaps=bundlecaps)
   >     bundler.newpart('output', data='changegroup2')
   >     bundler.newpart('changegroup', data=cg.getchunks())
   >