From patchwork Thu Feb 14 19:40:16 2013 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: [6,of,9] bundle-ng: move bundle generation to changegroup.py From: Sune Foldager X-Patchwork-Id: 995 Message-Id: <5a481bafae2d4e3ae66b.1360870816@firefly.edlund.dk> To: Mercurial Devel Date: Thu, 14 Feb 2013 20:40:16 +0100 # HG changeset patch # User Benoit Boissinot # Date 1360508600 -3600 # Node ID 5a481bafae2d4e3ae66b161b509cc7bb51a0560f # Parent 3821b9f543fbeabc29a2c34b3f78272837364586 bundle-ng: move bundle generation to changegroup.py diff -r 3821b9f543fb -r 5a481bafae2d mercurial/changegroup.py --- a/mercurial/changegroup.py Sun Feb 10 15:47:23 2013 +0100 +++ b/mercurial/changegroup.py Sun Feb 10 16:03:20 2013 +0100 @@ -289,7 +289,7 @@ yield self.close() - def generate(self, clnodes, getmfnodes, getfiles, getfilenodes, source): + def generate(self, commonrevs, clnodes, fastpathlinkrev, source): '''yield a sequence of changegroup chunks (strings)''' repo = self._repo cl = self._changelog @@ -298,6 +298,69 @@ progress = repo.ui.progress count = self.count _bundling = _('bundling') + _changesets = _('changesets') + _manifests = _('manifests') + _files = _('files') + + mfs = {} # needed manifests + fnodes = {} # needed file nodes + changedfiles = set() + fstate = ['', {}] + + # filter any nodes that claim to be part of the known set + def prune(revlog, missing): + rr, rl = revlog.rev, revlog.linkrev + return [n for n in missing + if rl(rr(n)) not in commonrevs] + + def lookup(revlog, x): + if revlog == cl: + c = cl.read(x) + changedfiles.update(c[3]) + mfs.setdefault(c[0], x) + count[0] += 1 + progress(_bundling, count[0], + unit=_changesets, total=count[1]) + return x + elif revlog == mf: + clnode = mfs[x] + if not fastpathlinkrev: + mdata = mf.readfast(x) + for f, n in mdata.iteritems(): + if f in changedfiles: + fnodes[f].setdefault(n, clnode) + count[0] += 1 + progress(_bundling, count[0], + unit=_manifests, total=count[1]) + return clnode + else: + progress(_bundling, count[0], item=fstate[0], + unit=_files, total=count[1]) + return fstate[1][x] + + self.start(lookup) + + def getmfnodes(): + for f in changedfiles: + fnodes[f] = {} + count[:] = [0, len(mfs)] + return prune(mf, mfs) + def getfiles(): + mfs.clear() + return changedfiles + def getfilenodes(fname, filerevlog): + if fastpathlinkrev: + ln, llr = filerevlog.node, filerevlog.linkrev + def genfilenodes(): + for r in filerevlog: + linkrev = llr(r) + if linkrev not in commonrevs: + yield filerevlog.node(r), cl.node(linkrev) + fnodes[fname] = dict(genfilenodes()) + fstate[0] = fname + fstate[1] = fnodes.pop(fname, {}) + return prune(filerevlog, fstate[1]) + count[:] = [0, len(clnodes)] for chunk in self.group(clnodes, cl, reorder=reorder): diff -r 3821b9f543fb -r 5a481bafae2d mercurial/localrepo.py --- a/mercurial/localrepo.py Sun Feb 10 15:47:23 2013 +0100 +++ b/mercurial/localrepo.py Sun Feb 10 16:03:20 2013 +0100 @@ -2029,122 +2029,14 @@ @unfilteredmethod def _changegroupsubset(self, commonrevs, csets, heads, bundler, source, fastpath=False): - cl = bundler._changelog - mf = bundler._manifest - mfs = {} # needed manifests - fnodes = {} # needed file nodes - changedfiles = set() - fstate = ['', {}] - count = [0, 0] - # can we go through the fast path ? heads.sort() fastpathlinkrev = fastpath or heads == sorted(self.heads()) self.hook('preoutgoing', throw=True, source=source) self.changegroupinfo(csets, source) - - # filter any nodes that claim to be part of the known set - def prune(revlog, missing): - rr, rl = revlog.rev, revlog.linkrev - return [n for n in missing - if rl(rr(n)) not in commonrevs] - - progress = self.ui.progress - _bundling = _('bundling') - _changesets = _('changesets') - _manifests = _('manifests') - _files = _('files') - - def lookup(revlog, x): - if revlog == cl: - c = cl.read(x) - changedfiles.update(c[3]) - mfs.setdefault(c[0], x) - count[0] += 1 - progress(_bundling, count[0], - unit=_changesets, total=count[1]) - return x - elif revlog == mf: - clnode = mfs[x] - if not fastpathlinkrev: - mdata = mf.readfast(x) - for f, n in mdata.iteritems(): - if f in changedfiles: - fnodes[f].setdefault(n, clnode) - count[0] += 1 - progress(_bundling, count[0], - unit=_manifests, total=count[1]) - return clnode - else: - progress(_bundling, count[0], item=fstate[0], - unit=_files, total=count[1]) - return fstate[1][x] - - bundler.start(lookup) - reorder = self.ui.config('bundle', 'reorder', 'auto') - if reorder == 'auto': - reorder = None - else: - reorder = util.parsebool(reorder) - - def gengroup(): - # Create a changenode group generator that will call our functions - # back to lookup the owning changenode and collect information. - count[:] = [0, len(csets)] - for chunk in bundler.group(csets, cl, reorder=reorder): - yield chunk - progress(_bundling, None) - - # Create a generator for the manifestnodes that calls our lookup - # and data collection functions back. - for f in changedfiles: - fnodes[f] = {} - count[:] = [0, len(mfs)] - for chunk in bundler.group(prune(mf, mfs), mf, reorder=reorder): - yield chunk - progress(_bundling, None) - - mfs.clear() - return changedfiles - def getfilenodes(fname, filerevlog): - if fastpathlinkrev: - ln, llr = filerevlog.node, filerevlog.linkrev - def genfilenodes(): - for r in filerevlog: - linkrev = llr(r) - if linkrev not in commonrevs: - yield filerevlog.node(r), cl.node(linkrev) - fnodes[fname] = dict(genfilenodes()) - fstate[0] = fname - fstate[1] = fnodes.pop(fname, {}) - return prune(filerevlog, fstate[1]) - - # Go through all our files in order sorted by name. - count[:] = [0, len(changedfiles)] - for fname in sorted(changedfiles): - filerevlog = self.file(fname) - if not len(filerevlog): - raise util.Abort(_("empty or missing revlog for %s") - % fname) - fstate[0] = fname - fstate[1] = fnodes.pop(fname, {}) - - nodelist = prune(filerevlog, fstate[1]) - if nodelist: - count[0] += 1 - yield bundler.fileheader(fname) - for chunk in bundler.group(nodelist, filerevlog, reorder): - yield chunk - - # Signal that no more groups are left. - yield bundler.close() - progress(_bundling, None) - - if csets: - self.hook('outgoing', node=hex(csets[0]), source=source) - - return changegroup.unbundle10(util.chunkbuffer(gengroup()), 'UN') + gengroup = bundler.generate(commonrevs, csets, fastpathlinkrev, source) + return changegroup.unbundle10(util.chunkbuffer(gengroup), 'UN') def changegroup(self, basenodes, source): # to avoid a race we use changegroupsubset() (issue1320)