Patchwork [5,of,9] bundle-ng: unify _changegroup and _changegroupsubset

login
register
mail settings
Submitter Sune Foldager
Date Feb. 14, 2013, 11:07 p.m.
Message ID <6599871bf18242484de2.1360883243@firefly.edlund.dk>
Download mbox | patch
Permalink /patch/1003/
State Superseded, archived
Commit 627cd7842e5d5c8535c69fe1f9d17418d6d1e056
Headers show

Comments

Sune Foldager - Feb. 14, 2013, 11:07 p.m.
# HG changeset patch
# User Benoit Boissinot <benoit.boissinot@ens-lyon.org>
# Date 1360507643 -3600
# Node ID 6599871bf18242484de2a8245bde12769ee8c7db
# Parent  6dd1df40c7389a645b5bb81aca15feb366653cca
bundle-ng: unify _changegroup and _changegroupsubset
Pierre-Yves David - March 12, 2013, 4:47 p.m.
On Fri, Feb 15, 2013 at 12:07:23AM +0100, Sune Foldager wrote:
> # HG changeset patch
> # User Benoit Boissinot <benoit.boissinot@ens-lyon.org>
> # Date 1360507643 -3600
> # Node ID 6599871bf18242484de2a8245bde12769ee8c7db
> # Parent  6dd1df40c7389a645b5bb81aca15feb366653cca
> bundle-ng: unify _changegroup and _changegroupsubset

Can you elaborate a bit? ;-)

> 
> diff -r 6dd1df40c738 -r 6599871bf182 mercurial/localrepo.py
> --- a/mercurial/localrepo.py	Sun Feb 10 01:12:44 2013 +0100
> +++ b/mercurial/localrepo.py	Sun Feb 10 15:47:23 2013 +0100
> @@ -1828,8 +1828,12 @@
>                          # push everything,
>                          # use the fast path, no race possible on push
>                          bundler = changegroup.bundle10(bundlecaps, self)
> -                        cg = self._changegroup(outgoing.missing, bundler,
> -                                               'push')
> +                        cg = self._changegroupsubset(outgoing.common,
> +                                                     outgoing.missing,
> +                                                     outgoing.missingheads,
> +                                                     bundler,
> +                                                     'push',
> +                                                     fastpath=True)

Would it worth it to have _changegroupsubset take an outgoing instance for all call?

This mean forging an outgoing object when calling repo.changegroupsubset

>                      else:
>                          cg = self.getlocalbundle('push', outgoing, bundlecaps)
>  
> @@ -2023,8 +2027,8 @@
>                                     bundlecaps=bundlecaps)
>  
>      @unfilteredmethod
> -    def _changegroupsubset(self, commonrevs, csets, heads, bundler, source):
> -
> +    def _changegroupsubset(self, commonrevs, csets, heads, bundler, source,
> +                           fastpath=False):
>          cl = bundler._changelog
>          mf = bundler._manifest
>          mfs = {} # needed manifests
> @@ -2034,10 +2038,8 @@
>  
>          # can we go through the fast path ?
>          heads.sort()
> -        if heads == sorted(self.heads()):
> -            return self._changegroup(csets, bundler, source)
> +        fastpathlinkrev = fastpath or heads == sorted(self.heads())

This only works when "self.filtername is None". I'm pretty sure it's always the
case for now, but this may change in the future. wW must protect ourself again
filtering.

Patch

diff -r 6dd1df40c738 -r 6599871bf182 mercurial/localrepo.py
--- a/mercurial/localrepo.py	Sun Feb 10 01:12:44 2013 +0100
+++ b/mercurial/localrepo.py	Sun Feb 10 15:47:23 2013 +0100
@@ -1828,8 +1828,12 @@ 
                         # push everything,
                         # use the fast path, no race possible on push
                         bundler = changegroup.bundle10(bundlecaps, self)
-                        cg = self._changegroup(outgoing.missing, bundler,
-                                               'push')
+                        cg = self._changegroupsubset(outgoing.common,
+                                                     outgoing.missing,
+                                                     outgoing.missingheads,
+                                                     bundler,
+                                                     'push',
+                                                     fastpath=True)
                     else:
                         cg = self.getlocalbundle('push', outgoing, bundlecaps)
 
@@ -2023,8 +2027,8 @@ 
                                    bundlecaps=bundlecaps)
 
     @unfilteredmethod
-    def _changegroupsubset(self, commonrevs, csets, heads, bundler, source):
-
+    def _changegroupsubset(self, commonrevs, csets, heads, bundler, source,
+                           fastpath=False):
         cl = bundler._changelog
         mf = bundler._manifest
         mfs = {} # needed manifests
@@ -2034,10 +2038,8 @@ 
 
         # can we go through the fast path ?
         heads.sort()
-        if heads == sorted(self.heads()):
-            return self._changegroup(csets, bundler, source)
+        fastpathlinkrev = fastpath or heads == sorted(self.heads())
 
-        # slow path
         self.hook('preoutgoing', throw=True, source=source)
         self.changegroupinfo(csets, source)
 
@@ -2065,10 +2067,11 @@ 
                 return x
             elif revlog == mf:
                 clnode = mfs[x]
-                mdata = mf.readfast(x)
-                for f, n in mdata.iteritems():
-                    if f in changedfiles:
-                        fnodes[f].setdefault(n, clnode)
+                if not fastpathlinkrev:
+                    mdata = mf.readfast(x)
+                    for f, n in mdata.iteritems():
+                        if f in changedfiles:
+                            fnodes[f].setdefault(n, clnode)
                 count[0] += 1
                 progress(_bundling, count[0],
                          unit=_manifests, total=count[1])
@@ -2089,6 +2092,14 @@ 
             mfs.clear()
             return changedfiles
         def getfilenodes(fname, filerevlog):
+            if fastpathlinkrev:
+                ln, llr = filerevlog.node, filerevlog.linkrev
+                def genfilenodes():
+                    for r in filerevlog:
+                        linkrev = llr(r)
+                        if linkrev not in commonrevs:
+                            yield filerevlog.node(r), cl.node(linkrev)
+                fnodes[fname] = dict(genfilenodes())
             fstate[0] = fname
             fstate[1] = fnodes.pop(fname, {})
             return prune(filerevlog, fstate[1])
@@ -2102,73 +2113,6 @@ 
         return self.changegroupsubset(basenodes, self.heads(), source)
 
     @unfilteredmethod
-    def _changegroup(self, nodes, bundler, source):
-        """Compute the changegroup of all nodes that we have that a recipient
-        doesn't.  Return a chunkbuffer object whose read() method will return
-        successive changegroup chunks.
-
-        This is much easier than the previous function as we can assume that
-        the recipient has any changenode we aren't sending them.
-
-        nodes is the set of nodes to send"""
-
-        cl = bundler._changelog
-        mf = bundler._manifest
-        mfs = {}
-        changedfiles = set()
-        fstate = ['']
-
-        self.hook('preoutgoing', throw=True, source=source)
-        self.changegroupinfo(nodes, source)
-
-        revset = set([cl.rev(n) for n in nodes])
-
-        def gennodelst(log):
-            ln, llr = log.node, log.linkrev
-            return [ln(r) for r in log if llr(r) in revset]
-
-        progress = self.ui.progress
-        _bundling = _('bundling')
-        _changesets = _('changesets')
-        _manifests = _('manifests')
-        _files = _('files')
-
-        def lookup(revlog, x):
-            count = bundler.count
-            if revlog == cl:
-                c = cl.read(x)
-                changedfiles.update(c[3])
-                mfs.setdefault(c[0], x)
-                count[0] += 1
-                progress(_bundling, count[0],
-                         unit=_changesets, total=count[1])
-                return x
-            elif revlog == mf:
-                count[0] += 1
-                progress(_bundling, count[0],
-                         unit=_manifests, total=count[1])
-                return cl.node(revlog.linkrev(revlog.rev(x)))
-            else:
-                progress(_bundling, count[0], item=fstate[0],
-                    total=count[1], unit=_files)
-                return cl.node(revlog.linkrev(revlog.rev(x)))
-
-        bundler.start(lookup)
-
-        def getmfnodes():
-            bundler.count[:] = [0, len(mfs)]
-            return gennodelst(mf)
-        def getfiles():
-            return changedfiles
-        def getfilenodes(fname, filerevlog):
-            fstate[0] = fname
-            return gennodelst(filerevlog)
-
-        gengroup = bundler.generate(nodes, getmfnodes, getfiles, getfilenodes,
-                                    source)
-        return changegroup.unbundle10(util.chunkbuffer(gengroup), 'UN')
-
-    @unfilteredmethod
     def addchangegroup(self, source, srctype, url, emptyok=False):
         """Add the changegroup returned by source.read() to this repo.
         srctype is a string like 'push', 'pull', or 'unbundle'.  url is