From patchwork Thu Feb 14 23:07:22 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: [4,of,9] bundle-ng: move gengroup into bundler From: Sune Foldager X-Patchwork-Id: 1001 Message-Id: <6dd1df40c7389a645b5b.1360883242@firefly.edlund.dk> To: Mercurial Devel Date: Fri, 15 Feb 2013 00:07:22 +0100 # HG changeset patch # User Sune Foldager # Date 1360455164 -3600 # Node ID 6dd1df40c7389a645b5bb81aca15feb366653cca # Parent ad5a6d13a57e28c75167facacc2811cefc014155 bundle-ng: move gengroup into bundler diff -r ad5a6d13a57e -r 6dd1df40c738 mercurial/changegroup.py --- a/mercurial/changegroup.py Sat Feb 09 23:42:03 2013 +0100 +++ b/mercurial/changegroup.py Sun Feb 10 01:12:44 2013 +0100 @@ -6,7 +6,7 @@ # GNU General Public License version 2 or any later version. from i18n import _ -from node import nullrev +from node import nullrev, hex import mdiff, util, dagutil import struct, os, bz2, zlib, tempfile @@ -225,11 +225,24 @@ class bundle10(object): deltaheader = _BUNDLE10_DELTA_HEADER - def __init__(self, bundlecaps=None): + def __init__(self, bundlecaps=None, repo=None): # Set of capabilities we can use to build the bundle. if bundlecaps is None: bundlecaps = set() self._bundlecaps = bundlecaps + if repo is not None: + self._changelog = repo.changelog + self._manifest = repo.manifest + reorder = repo.ui.config('bundle', 'reorder', 'auto') + if reorder == 'auto': + reorder = None + else: + reorder = util.parsebool(reorder) + else: + reorder = False + self._repo = repo + self._reorder = reorder + self.count = [0, 0] def start(self, lookup): self._lookup = lookup def close(self): @@ -276,6 +289,43 @@ yield self.close() + def generate(self, clnodes, getmfnodes, getfiles, getfilenodes, source): + '''yield a sequence of changegroup chunks (strings)''' + repo = self._repo + cl = self._changelog + mf = self._manifest + reorder = self._reorder + progress = repo.ui.progress + count = self.count + _bundling = _('bundling') + + count[:] = [0, len(clnodes)] + for chunk in self.group(clnodes, cl, reorder=reorder): + yield chunk + progress(_bundling, None) + + for chunk in self.group(getmfnodes(), mf, reorder=reorder): + yield chunk + progress(_bundling, None) + + changedfiles = getfiles() + count[:] = [0, len(changedfiles)] + for fname in sorted(changedfiles): + filerevlog = repo.file(fname) + if not len(filerevlog): + raise util.Abort(_("empty or missing revlog for %s") + % fname) + nodelist = getfilenodes(fname, filerevlog) + if nodelist: + count[0] += 1 + yield self.fileheader(fname) + for chunk in self.group(nodelist, filerevlog, reorder): + yield chunk + yield self.close() + progress(_bundling, None) + + if clnodes: + repo.hook('outgoing', node=hex(clnodes[0]), source=source) def revchunk(self, revlog, rev, prev): node = revlog.node(rev) diff -r ad5a6d13a57e -r 6dd1df40c738 mercurial/localrepo.py --- a/mercurial/localrepo.py Sat Feb 09 23:42:03 2013 +0100 +++ b/mercurial/localrepo.py Sun Feb 10 01:12:44 2013 +0100 @@ -1827,7 +1827,7 @@ if revs is None and not outgoing.excluded: # push everything, # use the fast path, no race possible on push - bundler = changegroup.bundle10(bundlecaps) + bundler = changegroup.bundle10(bundlecaps, self) cg = self._changegroup(outgoing.missing, bundler, 'push') else: @@ -1984,7 +1984,7 @@ csets, bases, heads = cl.nodesbetween(bases, heads) # We assume that all ancestors of bases are known common = cl.ancestors([cl.rev(n) for n in bases]) - bundler = changegroup.bundle10() + bundler = changegroup.bundle10(repo=self) return self._changegroupsubset(common, csets, heads, bundler, source) def getlocalbundle(self, source, outgoing, bundlecaps=None): @@ -1994,7 +1994,7 @@ precomputed sets in outgoing.""" if not outgoing.missing: return None - bundler = changegroup.bundle10(bundlecaps) + bundler = changegroup.bundle10(bundlecaps, self) return self._changegroupsubset(outgoing.common, outgoing.missing, outgoing.missingheads, @@ -2025,13 +2025,12 @@ @unfilteredmethod def _changegroupsubset(self, commonrevs, csets, heads, bundler, source): - cl = self.changelog - mf = self.manifest + cl = bundler._changelog + mf = bundler._manifest mfs = {} # needed manifests fnodes = {} # needed file nodes changedfiles = set() fstate = ['', {}] - count = [0, 0] # can we go through the fast path ? heads.sort() @@ -2055,6 +2054,7 @@ _files = _('files') def lookup(revlog, x): + count = bundler.count if revlog == cl: c = cl.read(x) changedfiles.update(c[3]) @@ -2079,56 +2079,23 @@ return fstate[1][x] bundler.start(lookup) - reorder = self.ui.config('bundle', 'reorder', 'auto') - if reorder == 'auto': - reorder = None - else: - reorder = util.parsebool(reorder) - def gengroup(): - # Create a changenode group generator that will call our functions - # back to lookup the owning changenode and collect information. - count[:] = [0, len(csets)] - for chunk in bundler.group(csets, cl, reorder=reorder): - yield chunk - progress(_bundling, None) - - # Create a generator for the manifestnodes that calls our lookup - # and data collection functions back. + def getmfnodes(): for f in changedfiles: fnodes[f] = {} - count[:] = [0, len(mfs)] - for chunk in bundler.group(prune(mf, mfs), mf, reorder=reorder): - yield chunk - progress(_bundling, None) + bundler.count[:] = [0, len(mfs)] + return prune(mf, mfs) + def getfiles(): + mfs.clear() + return changedfiles + def getfilenodes(fname, filerevlog): + fstate[0] = fname + fstate[1] = fnodes.pop(fname, {}) + return prune(filerevlog, fstate[1]) - mfs.clear() - - # Go through all our files in order sorted by name. - count[:] = [0, len(changedfiles)] - for fname in sorted(changedfiles): - filerevlog = self.file(fname) - if not len(filerevlog): - raise util.Abort(_("empty or missing revlog for %s") - % fname) - fstate[0] = fname - fstate[1] = fnodes.pop(fname, {}) - - nodelist = prune(filerevlog, fstate[1]) - if nodelist: - count[0] += 1 - yield bundler.fileheader(fname) - for chunk in bundler.group(nodelist, filerevlog, reorder): - yield chunk - - # Signal that no more groups are left. - yield bundler.close() - progress(_bundling, None) - - if csets: - self.hook('outgoing', node=hex(csets[0]), source=source) - - return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN') + gengroup = bundler.generate(csets, getmfnodes, getfiles, getfilenodes, + source) + return changegroup.unbundle10(util.chunkbuffer(gengroup), 'UN') def changegroup(self, basenodes, source): # to avoid a race we use changegroupsubset() (issue1320) @@ -2145,12 +2112,11 @@ nodes is the set of nodes to send""" - cl = self.changelog - mf = self.manifest + cl = bundler._changelog + mf = bundler._manifest mfs = {} changedfiles = set() fstate = [''] - count = [0, 0] self.hook('preoutgoing', throw=True, source=source) self.changegroupinfo(nodes, source) @@ -2168,6 +2134,7 @@ _files = _('files') def lookup(revlog, x): + count = bundler.count if revlog == cl: c = cl.read(x) changedfiles.update(c[3]) @@ -2187,46 +2154,19 @@ return cl.node(revlog.linkrev(revlog.rev(x))) bundler.start(lookup) - reorder = self.ui.config('bundle', 'reorder', 'auto') - if reorder == 'auto': - reorder = None - else: - reorder = util.parsebool(reorder) - def gengroup(): - '''yield a sequence of changegroup chunks (strings)''' - # construct a list of all changed files + def getmfnodes(): + bundler.count[:] = [0, len(mfs)] + return gennodelst(mf) + def getfiles(): + return changedfiles + def getfilenodes(fname, filerevlog): + fstate[0] = fname + return gennodelst(filerevlog) - count[:] = [0, len(nodes)] - for chunk in bundler.group(nodes, cl, reorder=reorder): - yield chunk - progress(_bundling, None) - - count[:] = [0, len(mfs)] - for chunk in bundler.group(gennodelst(mf), mf, reorder=reorder): - yield chunk - progress(_bundling, None) - - count[:] = [0, len(changedfiles)] - for fname in sorted(changedfiles): - filerevlog = self.file(fname) - if not len(filerevlog): - raise util.Abort(_("empty or missing revlog for %s") - % fname) - fstate[0] = fname - nodelist = gennodelst(filerevlog) - if nodelist: - count[0] += 1 - yield bundler.fileheader(fname) - for chunk in bundler.group(nodelist, filerevlog, reorder): - yield chunk - yield bundler.close() - progress(_bundling, None) - - if nodes: - self.hook('outgoing', node=hex(nodes[0]), source=source) - - return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN') + gengroup = bundler.generate(nodes, getmfnodes, getfiles, getfilenodes, + source) + return changegroup.unbundle10(util.chunkbuffer(gengroup), 'UN') @unfilteredmethod def addchangegroup(self, source, srctype, url, emptyok=False):