| Submitter | Augie Fackler |
|---|---|
| Date | March 13, 2014, 2:58 a.m. |
| Message ID | <c51e6e04d8c581e6d710.1394679536@augie-macbookair.alto.octopoid.net> |
| Download | mbox | patch |
| Permalink | /patch/3934/ |
| State | Rejected |
| Headers | show |
Comments
Messed up a for loop in my shell, drop this one for now please. On Wed, Mar 12, 2014 at 7:58 PM, Augie Fackler <raf@durin42.com> wrote: > # HG changeset patch > # User Augie Fackler <raf@durin42.com> > # Date 1380062977 14400 > # Tue Sep 24 18:49:37 2013 -0400 > # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5 > # Parent 1cd5bff45db28150d7c140be493fe851e6560f27 > WIP: lookaside clone extension > > diff --git a/hgext/lookaside.py b/hgext/lookaside.py > new file mode 100644 > --- /dev/null > +++ b/hgext/lookaside.py > @@ -0,0 +1,53 @@ > +"""Hacky proof of concept for lookaside bundle. > + > +Right now this doesn't do anything clever about bundle generation or > +distribution. Put a bundle file (generated with 'hg bundle --all' at a > +URL that urllib2 can fetch, then put that url in > +.hg/durin42-lookaside-location. > + > +This might mulch your data somehow. It's relatively untested, but it > +proves the concept. > +""" > + > +import shutil > +import urllib2 > + > +from mercurial import changegroup > +from mercurial import extensions > +from mercurial import wireproto > + > +def addcap(orig, *args, **kwargs): > + c = orig(*args, **kwargs) > + return c + ' x-durin42-lookaside' > + > +extensions.wrapfunction(wireproto, 'capabilities', addcap) > +wireproto.commands['capabilities'] = wireproto.capabilities, '' > + > +def lookaside(repo, proto): > + try: > + with repo.opener('durin42-lookaside-location') as f: > + return f.read().strip() > + except IOError: > + return '' > + > + > +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, '' > + > +def reposetup(ui, repo): > + origcls = repo.__class__ > + > + class lookasiderepo(origcls): > + > + def clone(self, remote, heads=[], stream=False): > + if not heads and remote.capable('x-durin42-lookaside'): > + url = remote._call('x-durin42-lookaside-clone') > + if url: > + ui.debug('fetching lookaside bundle %s\n' % url) > + # TODO: support resuming the bundle stream if a > connection fails > + self.addchangegroup( > + changegroup.readbundle(urllib2.urlopen(url), > 'lookaside bundle'), > + 'unbundle', 'bundle:lookaside bundle') > + return self.pull(remote, heads) > + return origcls.clone(self, remote, heads, stream=False) > + > + repo.__class__ = lookasiderepo > _______________________________________________ > Mercurial-devel mailing list > Mercurial-devel@selenic.com > http://selenic.com/mailman/listinfo/mercurial-devel >
And marmoute just gave me a great idea: we can do this using bundle2, by having the extension produce a header in the bundle that says "go fetch this URL, apply it, and then apply the stuff I gave you" On Wed, Mar 12, 2014 at 8:01 PM, Augie Fackler <raf@durin42.com> wrote: > Messed up a for loop in my shell, drop this one for now please. > > > On Wed, Mar 12, 2014 at 7:58 PM, Augie Fackler <raf@durin42.com> wrote: > >> # HG changeset patch >> # User Augie Fackler <raf@durin42.com> >> # Date 1380062977 14400 >> # Tue Sep 24 18:49:37 2013 -0400 >> # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5 >> # Parent 1cd5bff45db28150d7c140be493fe851e6560f27 >> WIP: lookaside clone extension >> >> diff --git a/hgext/lookaside.py b/hgext/lookaside.py >> new file mode 100644 >> --- /dev/null >> +++ b/hgext/lookaside.py >> @@ -0,0 +1,53 @@ >> +"""Hacky proof of concept for lookaside bundle. >> + >> +Right now this doesn't do anything clever about bundle generation or >> +distribution. Put a bundle file (generated with 'hg bundle --all' at a >> +URL that urllib2 can fetch, then put that url in >> +.hg/durin42-lookaside-location. >> + >> +This might mulch your data somehow. It's relatively untested, but it >> +proves the concept. >> +""" >> + >> +import shutil >> +import urllib2 >> + >> +from mercurial import changegroup >> +from mercurial import extensions >> +from mercurial import wireproto >> + >> +def addcap(orig, *args, **kwargs): >> + c = orig(*args, **kwargs) >> + return c + ' x-durin42-lookaside' >> + >> +extensions.wrapfunction(wireproto, 'capabilities', addcap) >> +wireproto.commands['capabilities'] = wireproto.capabilities, '' >> + >> +def lookaside(repo, proto): >> + try: >> + with repo.opener('durin42-lookaside-location') as f: >> + return f.read().strip() >> + except IOError: >> + return '' >> + >> + >> +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, '' >> + >> +def reposetup(ui, repo): >> + origcls = repo.__class__ >> + >> + class lookasiderepo(origcls): >> + >> + def clone(self, remote, heads=[], stream=False): >> + if not heads and remote.capable('x-durin42-lookaside'): >> + url = remote._call('x-durin42-lookaside-clone') >> + if url: >> + ui.debug('fetching lookaside bundle %s\n' % url) >> + # TODO: support resuming the bundle stream if a >> connection fails >> + self.addchangegroup( >> + changegroup.readbundle(urllib2.urlopen(url), >> 'lookaside bundle'), >> + 'unbundle', 'bundle:lookaside bundle') >> + return self.pull(remote, heads) >> + return origcls.clone(self, remote, heads, stream=False) >> + >> + repo.__class__ = lookasiderepo >> _______________________________________________ >> Mercurial-devel mailing list >> Mercurial-devel@selenic.com >> http://selenic.com/mailman/listinfo/mercurial-devel >> > >
I took the idea of this extension and built it out into a more finished implementation, which I'm calling "bundleclone." https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone We may deploy this at Mozilla to reduce server load. More details at https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 I'd appreciate any feedback. On 3/12/14, 9:58 PM, Augie Fackler wrote: > # HG changeset patch > # User Augie Fackler <raf@durin42.com> > # Date 1380062977 14400 > # Tue Sep 24 18:49:37 2013 -0400 > # Node ID c51e6e04d8c581e6d7109fb3a54a69121c12d7e5 > # Parent 1cd5bff45db28150d7c140be493fe851e6560f27 > WIP: lookaside clone extension > > diff --git a/hgext/lookaside.py b/hgext/lookaside.py > new file mode 100644 > --- /dev/null > +++ b/hgext/lookaside.py > @@ -0,0 +1,53 @@ > +"""Hacky proof of concept for lookaside bundle. > + > +Right now this doesn't do anything clever about bundle generation or > +distribution. Put a bundle file (generated with 'hg bundle --all' at a > +URL that urllib2 can fetch, then put that url in > +.hg/durin42-lookaside-location. > + > +This might mulch your data somehow. It's relatively untested, but it > +proves the concept. > +""" > + > +import shutil > +import urllib2 > + > +from mercurial import changegroup > +from mercurial import extensions > +from mercurial import wireproto > + > +def addcap(orig, *args, **kwargs): > + c = orig(*args, **kwargs) > + return c + ' x-durin42-lookaside' > + > +extensions.wrapfunction(wireproto, 'capabilities', addcap) > +wireproto.commands['capabilities'] = wireproto.capabilities, '' > + > +def lookaside(repo, proto): > + try: > + with repo.opener('durin42-lookaside-location') as f: > + return f.read().strip() > + except IOError: > + return '' > + > + > +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, '' > + > +def reposetup(ui, repo): > + origcls = repo.__class__ > + > + class lookasiderepo(origcls): > + > + def clone(self, remote, heads=[], stream=False): > + if not heads and remote.capable('x-durin42-lookaside'): > + url = remote._call('x-durin42-lookaside-clone') > + if url: > + ui.debug('fetching lookaside bundle %s\n' % url) > + # TODO: support resuming the bundle stream if a connection fails > + self.addchangegroup( > + changegroup.readbundle(urllib2.urlopen(url), 'lookaside bundle'), > + 'unbundle', 'bundle:lookaside bundle') > + return self.pull(remote, heads) > + return origcls.clone(self, remote, heads, stream=False) > + > + repo.__class__ = lookasiderepo > _______________________________________________ > Mercurial-devel mailing list > Mercurial-devel@selenic.com > http://selenic.com/mailman/listinfo/mercurial-devel >
On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: > I took the idea of this extension and built it out into a more finished > implementation, which I'm calling "bundleclone." > > https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone > > We may deploy this at Mozilla to reduce server load. More details at > https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 > > I'd appreciate any feedback. Can I get you to make a wiki page for it?
On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: > I took the idea of this extension and built it out into a more finished > implementation, which I'm calling "bundleclone." > > https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone > > We may deploy this at Mozilla to reduce server load. More details at > https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 > > I'd appreciate any feedback. Can I get you to make a wiki page for it?
On 7/19/14, 3:25 PM, Matt Mackall wrote: > On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: >> I took the idea of this extension and built it out into a more finished >> implementation, which I'm calling "bundleclone." >> >> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone >> >> We may deploy this at Mozilla to reduce server load. More details at >> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 >> >> I'd appreciate any feedback. > > Can I get you to make a wiki page for it? http://mercurial.selenic.com/wiki/BundleCloneExtension I also backported the extension to work with 2.5 and above.
On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote: > On 7/19/14, 3:25 PM, Matt Mackall wrote: >> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: >>> I took the idea of this extension and built it out into a more finished >>> implementation, which I'm calling "bundleclone." >>> >>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone >>> >>> We may deploy this at Mozilla to reduce server load. More details at >>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 >>> >>> I'd appreciate any feedback. >> >> Can I get you to make a wiki page for it? > > http://mercurial.selenic.com/wiki/BundleCloneExtension > > I also backported the extension to work with 2.5 and above. > I have a few code review comments. How would you like them delivered?
On 7/21/14, 8:42 AM, Augie Fackler wrote: > > On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote: > >> On 7/19/14, 3:25 PM, Matt Mackall wrote: >>> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: >>>> I took the idea of this extension and built it out into a more finished >>>> implementation, which I'm calling "bundleclone." >>>> >>>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone >>>> >>>> We may deploy this at Mozilla to reduce server load. More details at >>>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 >>>> >>>> I'd appreciate any feedback. >>> >>> Can I get you to make a wiki page for it? >> >> http://mercurial.selenic.com/wiki/BundleCloneExtension >> >> I also backported the extension to work with 2.5 and above. >> > > I have a few code review comments. How would you like them delivered? Email to me is fine. I'll get a Bugzilla component created and update buglink in the extension sometime this week.
On Jul 21, 2014, at 10:20 AM, Gregory Szorc <gregory.szorc@gmail.com> wrote: > On 7/21/14, 8:42 AM, Augie Fackler wrote: >> >> On Jul 20, 2014, at 8:30 PM, Gregory Szorc <gregory.szorc@gmail.com> wrote: >> >>> On 7/19/14, 3:25 PM, Matt Mackall wrote: >>>> On Sat, 2014-07-19 at 15:00 -0500, Gregory Szorc wrote: >>>>> I took the idea of this extension and built it out into a more finished >>>>> implementation, which I'm calling "bundleclone." >>>>> >>>>> https://hg.mozilla.org/hgcustom/version-control-tools/file/default/hgext/bundleclone >>>>> >>>>> We may deploy this at Mozilla to reduce server load. More details at >>>>> https://bugzilla.mozilla.org/show_bug.cgi?id=1041173 >>>>> >>>>> I'd appreciate any feedback. >>>> >>>> Can I get you to make a wiki page for it? >>> >>> http://mercurial.selenic.com/wiki/BundleCloneExtension >>> >>> I also backported the extension to work with 2.5 and above. >>> >> >> I have a few code review comments. How would you like them delivered? > > Email to me is fine. I'll get a Bugzilla component created and update buglink in the extension sometime this week. Will do. On second look-through, I don't have any immediate comments.
Patch
diff --git a/hgext/lookaside.py b/hgext/lookaside.py new file mode 100644 --- /dev/null +++ b/hgext/lookaside.py @@ -0,0 +1,53 @@ +"""Hacky proof of concept for lookaside bundle. + +Right now this doesn't do anything clever about bundle generation or +distribution. Put a bundle file (generated with 'hg bundle --all' at a +URL that urllib2 can fetch, then put that url in +.hg/durin42-lookaside-location. + +This might mulch your data somehow. It's relatively untested, but it +proves the concept. +""" + +import shutil +import urllib2 + +from mercurial import changegroup +from mercurial import extensions +from mercurial import wireproto + +def addcap(orig, *args, **kwargs): + c = orig(*args, **kwargs) + return c + ' x-durin42-lookaside' + +extensions.wrapfunction(wireproto, 'capabilities', addcap) +wireproto.commands['capabilities'] = wireproto.capabilities, '' + +def lookaside(repo, proto): + try: + with repo.opener('durin42-lookaside-location') as f: + return f.read().strip() + except IOError: + return '' + + +wireproto.commands['x-durin42-lookaside-clone'] = lookaside, '' + +def reposetup(ui, repo): + origcls = repo.__class__ + + class lookasiderepo(origcls): + + def clone(self, remote, heads=[], stream=False): + if not heads and remote.capable('x-durin42-lookaside'): + url = remote._call('x-durin42-lookaside-clone') + if url: + ui.debug('fetching lookaside bundle %s\n' % url) + # TODO: support resuming the bundle stream if a connection fails + self.addchangegroup( + changegroup.readbundle(urllib2.urlopen(url), 'lookaside bundle'), + 'unbundle', 'bundle:lookaside bundle') + return self.pull(remote, heads) + return origcls.clone(self, remote, heads, stream=False) + + repo.__class__ = lookasiderepo